// QueryParams.java
// -----------------------
// part of YACY
// (C) by Michael Peter Christen; mc@yacy.net
// first published on http://www.anomic.de
// Frankfurt, Germany, 2005
// Created: 10.10.2005
//
// $LastChangedDate$
// $LastChangedRevision$
// $LastChangedBy$
//
// This program is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

package net.yacy.search.query;

import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import org.apache.commons.lang3.StringUtils;
import org.apache.lucene.util.automaton.Automata;
import org.apache.lucene.util.automaton.Automaton;
import org.apache.lucene.util.automaton.RegExp;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.SolrQuery.SortClause;
import org.apache.solr.common.params.DisMaxParams;
import org.apache.solr.common.params.FacetParams;

import net.yacy.cora.document.analysis.Classification;
import net.yacy.cora.document.analysis.Classification.ContentDomain;
import net.yacy.cora.document.encoding.ASCII;
import net.yacy.cora.document.id.AnchorURL;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.federate.solr.Ranking;
import net.yacy.cora.federate.yacy.CacheStrategy;
import net.yacy.cora.geo.GeoLocation;
import net.yacy.cora.lod.vocabulary.Tagging;
import net.yacy.cora.order.Base64Order;
import net.yacy.cora.protocol.RequestHeader;
import net.yacy.cora.storage.HandleSet;
import net.yacy.cora.util.ConcurrentLog;
import net.yacy.cora.util.SpaceExceededException;
import net.yacy.document.LibraryProvider;
import net.yacy.document.ProbabilisticClassifier;
import net.yacy.document.Tokenizer;
import net.yacy.kelondro.data.word.Word;
import net.yacy.kelondro.data.word.WordReferenceRow;
import net.yacy.kelondro.index.RowHandleSet;
import net.yacy.kelondro.util.Bitfield;
import net.yacy.kelondro.util.SetTools;
import net.yacy.peers.Seed;
import net.yacy.search.index.Segment;
import net.yacy.search.navigator.NavigatorPlugins;
import net.yacy.search.navigator.NavigatorSort;
import net.yacy.search.ranking.RankingProfile;
import net.yacy.search.schema.CollectionConfiguration;
import net.yacy.search.schema.CollectionSchema;

public final class QueryParams {

	/** The default max count of item lines in navigator */
    public static final int FACETS_STANDARD_MAXCOUNT_DEFAULT = 100;
    
    /** The default maximum number of date elements in the date navigator */
    public static final int FACETS_DATE_MAXCOUNT_DEFAULT = 640;
    
	/**
	 * The Solr facet limit to apply when resorting or filtering is done in a YaCy
	 * search navigator. For example sort by ascending counts or by descending
	 * indexed terms are not supported with Solr 6.6 (but should be on Solr 7 JSON
	 * Facet API - see
	 * https://lucene.apache.org/solr/guide/7_6/json-facet-api.html). The limit
	 * defined here is set large enough so that resorting can be done by the YaCy
	 * Navigator itself. We don't set the facet to unlimited to prevent a too high
	 * memory usage.
	 */
	private static final int FACETS_MAXCOUNT_FOR_RESORT_ON_SEARCH_NAV = 100000;
    
    public enum Searchdom {
        LOCAL, CLUSTER, GLOBAL;

        @Override
        public String toString() {
            if (this == LOCAL) return "local";
            else if (this == CLUSTER) return "global"; // yes thats right: global, not cluster because a cluster search is a global search
            else if (this == GLOBAL) return "global";
            return "local";
        }
    }

    private static final Map<String, CollectionSchema> defaultfacetfields = new HashMap<String, CollectionSchema>();
    static {
        // the key shall match with configuration property search.navigation
        // defaultfacetfields.put("location", CollectionSchema.coordinate_p_0_coordinate); // coordinate_p can't be used for facet (subfields), as value isn't used subfield can be used
        defaultfacetfields.put("hosts", CollectionSchema.host_s);
        defaultfacetfields.put("protocol", CollectionSchema.url_protocol_s);
        defaultfacetfields.put("filetype", CollectionSchema.url_file_ext_s);
        defaultfacetfields.put("date", CollectionSchema.dates_in_content_dts);
        defaultfacetfields.put("authors", CollectionSchema.author_sxt);
        defaultfacetfields.put("collections", CollectionSchema.collection_sxt);
        defaultfacetfields.put("language", CollectionSchema.language_s);
        //missing: namespace
    }
    
    /** List of Solr fields used to extract text snippets when requesting the Solr index */
    private final static CollectionSchema[] SOLR_SNIPPET_FIELDS = new CollectionSchema[]{CollectionSchema.description_txt, CollectionSchema.h4_txt, CollectionSchema.h3_txt, CollectionSchema.h2_txt, CollectionSchema.h1_txt, CollectionSchema.text_t};
    
    public static final Bitfield empty_constraint    = new Bitfield(4, "AAAAAA");
    public static final Pattern catchall_pattern = Pattern.compile(".*");

    private final QueryGoal queryGoal;
    public int itemsPerPage;
    public int offset;
    
    /** The URL mask pattern compiled from the urlMasString. 
     * Null when the urlMaskString is not user provided but generated from the query modifiers */
    public Pattern urlMaskPattern;
    public Automaton urlMaskAutomaton;
    public String urlMaskString;

    public final Pattern prefer;
    public final String tld, inlink;
    
    /** true when the urlMasString is just a catch all pattern such as ".*" */
    boolean urlMask_isCatchall;
    
    /** Content-Type classification of expected results */
    public final Classification.ContentDomain contentdom;
    
	/**
	 * <p>When false, results can be extended to documents including links to documents
	 * of {@link #contentdom} type, whithout being themselves of that type.</p>
	 * Examples :
	 * <ul>
	 * <li>contentdom == IMAGE, strictContentDom == true
	 *  <ul>
	 *   <li>jpeg image : acceptable result</li>
	 * 	 <li>html page embedding images : rejected</li>
	 *  </ul>
	 * </li>
	 * <li>contentdom == IMAGE, strictContentDom == false
	 *  <ul>
	 *   <li>jpeg image : acceptable result</li>
	 * 	 <li>html page embedding images : acceptable result</li>
	 *  </ul>
	 * </li>
	 * </ul> 
	 */
    private boolean strictContentDom = false;
    
	/**
	 * The maximum number of suggestions ("Did you mean") to display at the top of
	 * the first search results page
	 */
    private int maxSuggestions = 0;
    
    public final String targetlang;
    protected final Collection<Tagging.Metatag> metatags;
    public final Searchdom domType;
    private final int zonecode;
    public final int maxDistance;
    public final Bitfield constraint;
    public final boolean allofconstraint;
    protected CacheStrategy snippetCacheStrategy;
    private boolean snippetFetchFullText;
    private boolean localSolrOnly;
    public final RankingProfile ranking;
    private final Segment indexSegment;
    public final String clienthost; // this is the client host that starts the query, not a site operator
    protected final Set<String> siteexcludes; // set of domain hashes that are excluded if not included by sitehash
    public final QueryModifier modifier;
    public Seed remotepeer;
    public final long starttime; // the time when the query started, how long it should take and the time when the timeout is reached (milliseconds)
    protected final long maxtime;
    // values that are set after a search:
    public int transmitcount; // number of results that had been shown to the user
    public long searchtime, urlretrievaltime, snippetcomputationtime; // time to perform the search, to get all the urls, and to compute the snippets
    public final String userAgent;
    protected double lat, lon, radius;
    
    /** Map from facet/navigator name to sort properties */
    public Map<String, NavigatorSort> facetfields;
    private SolrQuery cachedQuery;
    private CollectionConfiguration solrSchema;
    public final int timezoneOffset;
    
    /** The max count of item lines in navigator */
    private int standardFacetsMaxCount;
    
    /** The maximum number of date elements in the date navigator */
    private int dateFacetMaxCount;
    

    public QueryParams(
        final QueryGoal queryGoal,
        final QueryModifier modifier,
        final int maxDistance,
        final String prefer,
        final ContentDomain contentdom,
        final String language,
        final int timezoneOffset,
        final Collection<Tagging.Metatag> metatags,
        final CacheStrategy snippetCacheStrategy,
        final int itemsPerPage,
        final int offset,
        final String urlMask,
        final String tld,
        final String inlink,
        final Searchdom domType,
        final Bitfield constraint,
        final boolean allofconstraint,
        final Set<String> siteexcludes,
        final int domainzone,
        final String host,
        final boolean specialRights,
        final Segment indexSegment,
        final RankingProfile ranking,
        final String userAgent,
        final double lat,
        final double lon,
        final double radius,
        final Set<String> navConfigs
        ) {
        this.queryGoal = queryGoal;
        this.modifier = modifier;
        this.ranking = ranking;
        this.maxDistance = maxDistance;
        this.contentdom = contentdom;
        this.timezoneOffset = timezoneOffset;
        this.itemsPerPage = Math.min((specialRights) ? 10000 : 1000, itemsPerPage);
        if(domType == Searchdom.LOCAL) {
        	/* No offset restriction on local index only requests, as only itemsPerPage will be loaded */
        	this.offset = Math.max(0, offset);
        } else {
        	/* Offset has to be limited on requests mixing local and remote results, because all results before offset are loaded */
        	this.offset = Math.max(0, Math.min((specialRights) ? 10000 - this.itemsPerPage : 1000 - this.itemsPerPage, offset));
        }
        try {
            this.urlMaskString = urlMask;
            // solr doesn't like slashes, backslashes or doublepoints; remove them // urlmask = ".*\\." + ft + "(\\?.*)?";
            int p;
            while ((p = this.urlMaskString.indexOf(':')) >= 0) this.urlMaskString = this.urlMaskString.substring(0, p) + "." + this.urlMaskString.substring(p + 1);
            while ((p = this.urlMaskString.indexOf('/')) >= 0) this.urlMaskString = this.urlMaskString.substring(0, p) + "." + this.urlMaskString.substring(p + 1);
            while ((p = this.urlMaskString.indexOf('\\')) >= 0) this.urlMaskString = this.urlMaskString.substring(0, p) + "." + this.urlMaskString.substring(p + 2);
            this.urlMaskAutomaton = Automata.makeString(this.urlMaskString);
            this.urlMaskPattern = Pattern.compile(this.urlMaskString);
        } catch (final Throwable ex) {
            throw new IllegalArgumentException("Not a valid regular expression: " + urlMask, ex);
        }
        this.urlMask_isCatchall = this.urlMaskString.equals(catchall_pattern.toString());
        if (this.urlMask_isCatchall) {
            final String filter = QueryParams.buildApproximateURLFilter(modifier, tld);
            if (!QueryParams.catchall_pattern.toString().equals(filter)) {
                this.urlMaskString = filter;
                this.urlMaskAutomaton = Automata.makeString(filter);
                this.urlMask_isCatchall = false;
                /* We let here the urlMaskPattern null :
                 * final URL match checking will be made with the more accurate matchesURL function */
                this.urlMaskPattern = null;
            }
        }
        this.tld = tld;
        this.inlink = inlink;
        try {
            this.prefer = Pattern.compile(prefer);
        } catch (final PatternSyntaxException ex) {
            throw new IllegalArgumentException("Not a valid regular expression: " + prefer, ex);
        }
        assert language != null;
        this.targetlang = language;
        this.metatags = metatags;
        this.domType = domType;
        this.zonecode = domainzone;
        this.constraint = constraint;
        this.allofconstraint = allofconstraint;
        this.siteexcludes = siteexcludes != null && siteexcludes.isEmpty() ? null: siteexcludes;
        this.snippetCacheStrategy = snippetCacheStrategy;
        this.snippetFetchFullText = false;
        this.localSolrOnly = false;
        this.clienthost = host;
        this.remotepeer = null;
        this.starttime = System.currentTimeMillis();
        this.maxtime = 10000;
        this.indexSegment = indexSegment;
        this.userAgent = userAgent;
        this.transmitcount = 0;
        // we normalize here the location and radius because that should cause a better caching
        // and as surplus it will increase privacy
        this.lat = Math.floor(lat * this.kmNormal) / this.kmNormal;
        this.lon = Math.floor(lon * this.kmNormal) / this.kmNormal;
        this.radius = Math.floor(radius * this.kmNormal + 1) / this.kmNormal;
        this.facetfields = new HashMap<>();
        
        this.solrSchema = indexSegment.fulltext().getDefaultConfiguration();
        for (final String navConfig: navConfigs) {
            CollectionSchema f = defaultfacetfields.get(NavigatorPlugins.getNavName(navConfig));
            // handle special field, authors_sxt (add to facet w/o contains check, as authors_sxt is not enabled (is copyfield))
            // dto. for coordinate_p_0_coordinate is not enabled but used for location facet (because coordinate_p not valid for facet field)
            if (f != null && (solrSchema.contains(f) || f == CollectionSchema.author_sxt || f == CollectionSchema.coordinate_p_0_coordinate))
                this.facetfields.put(f.getSolrFieldName(), NavigatorPlugins.parseNavSortConfig(navConfig));
        }
        if (LibraryProvider.autotagging != null) for (Tagging v: LibraryProvider.autotagging.getVocabularies()) {
            if (v.isFacet()) {
                this.facetfields.put(CollectionSchema.VOCABULARY_PREFIX + v.getName() + CollectionSchema.VOCABULARY_TERMS_SUFFIX, NavigatorSort.COUNT_DESC);
            }
        }
        for (String context: ProbabilisticClassifier.getContextNames()) {
            this.facetfields.put(CollectionSchema.VOCABULARY_PREFIX + context + CollectionSchema.VOCABULARY_TERMS_SUFFIX, NavigatorSort.COUNT_DESC);
        }
        this.cachedQuery = null;
        this.standardFacetsMaxCount = FACETS_STANDARD_MAXCOUNT_DEFAULT;
        this.dateFacetMaxCount = FACETS_DATE_MAXCOUNT_DEFAULT;
    }

	/**
	 * Generate an URL filter from the query modifier and eventual tld, usable as a
	 * first approximation for filtering, and compatible with the yacy/search
	 * API.<br/>
	 * For truly accurate filtering, checking constraints against parsed URLs in 
	 * MultiprotocolURL instances is easier and more reliable than building a complex regular
	 * expression that must be both compatible with the JDK {@link Pattern} and with Lucene {@link RegExp}.
	 * 
	 * @param modifier
	 *            query modifier with eventual protocol, sitehost and filetype
	 *            constraints. The modifier parameter itselft must not be null.
	 * @param tld
	 *            an eventual Top Level Domain name
	 * @return an URL filter regular expression from the provided modifier and tld
	 *         constraints, matching anything when there are no constraints at all.
	 */
	protected static String buildApproximateURLFilter(final QueryModifier modifier, final String tld) {
		final String protocolfilter = modifier.protocol == null ? ".*" : modifier.protocol;
		final String defaulthostprefix = "www";
		final String hostfilter;
		if(modifier.sitehost == null && tld == null) {
			hostfilter = ".*";
		} else if(modifier.sitehost == null) {
			hostfilter = ".*\\." + tld;
		} else if(modifier.sitehost.startsWith(defaulthostprefix + ".")){
			hostfilter = "(" + defaulthostprefix + "\\.)?" + modifier.sitehost.substring(4);
		} else {
			hostfilter = "(" + defaulthostprefix + "\\.)?" + modifier.sitehost;
		}
		final String filefilter = modifier.filetype == null ? ".*" : ".*" + modifier.filetype + ".*"; // TODO: should be ".ext" but while/comment above suggests not -> add filetype contrain pullOneFilteredFromRWI()
		String filter = protocolfilter + "..." + hostfilter + "." + filefilter;
        if (!filter.equals(".*....*..*")) {
        	/* Remove redundant sequences of catch all expressions */
            Pattern r = Pattern.compile("(\\.|(\\.\\*))\\.\\*");
            Matcher m;
            while ((m = r.matcher(filter)).find()) {
            	filter = m.replaceAll(".*");
            }
        } else {
			filter = QueryParams.catchall_pattern.toString();
		}
		return filter;
	}

    private double kmNormal = 100.d; // 100 =ca 40000.d / 360.d == 111.11 - if lat/lon is multiplied with this, rounded and diveded by this, the location is normalized to a 1km grid

    public Segment getSegment() {
        return this.indexSegment;
    }

    public int neededResults() {
        // the number of result lines that must be computed
        return this.offset + this.itemsPerPage;
    }

    public int itemsPerPage() {
        // the number of result lines that are displayed at once (size of result page)
        return this.itemsPerPage;
    }
    
    public void setOffset(final int newOffset) {
        this.offset = newOffset;
    }

    public boolean isLocal() {
        return this.domType == Searchdom.LOCAL;
    }
    
    /**
     * @return the max count of item lines in standard navigators
     */
    public int getStandardFacetsMaxCount() {
		return this.standardFacetsMaxCount;
	}
    
    /**
     * @param standardFacetsMaxCount the max count of item lines in standard navigators
     */
    public void setStandardFacetsMaxCount(final int standardFacetsMaxCount) {
		this.standardFacetsMaxCount = standardFacetsMaxCount;
	}
    
    /**
     * @return the maximum number of date elements in the date navigator
     */
    public int getDateFacetMaxCount() {
		return this.dateFacetMaxCount;
	}
    
    /**
     * @param dateFacetMaxCount the maximum number of date elements in the date navigator
     */
    public void setDateFacetMaxCount(final int dateFacetMaxCount) {
		this.dateFacetMaxCount = dateFacetMaxCount;
	}
    
    /**
     * @return false when results can be extended to documents including links to documents ot contentdom type.
     */
    public boolean isStrictContentDom() {
		return this.strictContentDom;
	}
    
    /**
     * @param strictContentDom when false, results can be extended to documents including links to documents ot contentdom type.
     */
    public void setStrictContentDom(final boolean strictContentDom) {
		this.strictContentDom = strictContentDom;
	}
    
	/**
	 * @return The maximum number of suggestions ("Did you mean") to display at the
	 *         top of the first search results page
	 */
	public int getMaxSuggestions() {
		return this.maxSuggestions;
	}

	/**
	 * @param maxSuggestions
	 *            The maximum number of suggestions ("Did you mean") to display at
	 *            the top of the first search results page
	 */
	public void setMaxSuggestions(final int maxSuggestions) {
		this.maxSuggestions = maxSuggestions;
	}

    public static HandleSet hashes2Set(final String query) {
        final HandleSet keyhashes = new RowHandleSet(WordReferenceRow.urlEntryRow.primaryKeyLength, WordReferenceRow.urlEntryRow.objectOrder, 0);
        if (query != null) {
            for (int i = 0; i < (query.length() / Word.commonHashLength); i++) try {
                keyhashes.put(ASCII.getBytes(query.substring(i * Word.commonHashLength, (i + 1) * Word.commonHashLength)));
            } catch (final SpaceExceededException e) {
                ConcurrentLog.logException(e);
            }
        }
        return keyhashes;
    }

    public static String hashSet2hashString(final HandleSet hashes) {
        final byte[] bb = new byte[hashes.size() * Word.commonHashLength];
        int p = 0;
        for (final byte[] b : hashes) {
            assert b.length == Word.commonHashLength : "hash = " + ASCII.String(b);
            System.arraycopy(b, 0, bb, p, Word.commonHashLength);
            p += Word.commonHashLength;
        }
        return ASCII.String(bb);
    }

    public static String hashSet2hashString(final Set<String> hashes) {
        final byte[] bb = new byte[hashes.size() * Word.commonHashLength];
        int p = 0;
        for (final String s : hashes) {
            assert s.length() == Word.commonHashLength : "hash = " + s;
            System.arraycopy(ASCII.getBytes(s), 0, bb, p, Word.commonHashLength);
            p += Word.commonHashLength;
        }
        return ASCII.String(bb);
    }

    public static String anonymizedQueryHashes(final HandleSet hashes) {
        // create a more anonymized representation of a query hashes for logging
        final Iterator<byte[]> i = hashes.iterator();
        final StringBuilder sb = new StringBuilder(hashes.size() * (Word.commonHashLength + 2) + 2);
        sb.append("[");
        byte[] hash;
        if (i.hasNext()) {
            hash = i.next();
            sb.append(ASCII.String(hash).substring(0, 3)).append(".........");
        }
        while (i.hasNext()) {
            hash = i.next();
            sb.append(", ").append(ASCII.String(hash).substring(0, 3)).append(".........");
        }
        sb.append("]");
        return sb.toString();
    }
    
	/**
	 * Check wheter the given URL matches the eventual modifier and top-level domain
	 * constraints. Should be preferred as more accurate than the url mask pattern generated with
	 * {@link #buildApproximateURLFilter(QueryModifier, String)}.
	 * 
	 * @param modifier
	 *            the query modifier with eventual constraints on protocoln, host
	 *            name or file extension
	 * @param tld
	 *            an eventual top-level domain name to filter on
	 * @param url
	 *            the url to check
	 * @return the constraint that did not match ("url" when url is null,
	 *         "protocol", "sitehost", "tld", or "filetype"), or the empty string
	 *         when the url matches
	 */
	public static String matchesURL(final QueryModifier modifier, final String tld, final MultiProtocolURL url) {
		if (url == null) {
			return "url";
		}
		if (modifier != null) {
			if (modifier.protocol != null) {
				if (!modifier.protocol.equalsIgnoreCase(url.getProtocol())) {
					return "protocol";
				}
			}
			if (modifier.sitehost != null) {
				/*
				 * consider to search for hosts with 'www'-prefix, if not already part of the
				 * host name
				 */
				final String wwwPrefix = "www.";
				final String host;
				final String hostWithWwwPrefix;
				if (modifier.sitehost.startsWith(wwwPrefix)) {
					hostWithWwwPrefix = modifier.sitehost;
					host = modifier.sitehost.substring(wwwPrefix.length());
				} else {
					hostWithWwwPrefix = wwwPrefix + modifier.sitehost;
					host = modifier.sitehost;
				}
				if (!host.equalsIgnoreCase(url.getHost()) && !hostWithWwwPrefix.equals(url.getHost())) {
					return "sitehost";
				}
			}
			if (tld != null) {
				if (!tld.equalsIgnoreCase(url.getTLD())) {
					return "tld";
				}
			}
			if (modifier.filetype != null) {
				if (!modifier.filetype.equalsIgnoreCase(MultiProtocolURL.getFileExtension(url.getFileName()))) {
					return "filetype";
				}
			}
		}
		return "";
	}

    /**
     * check if the given text matches with the query
     * this checks inclusion and exclusion words
     * @param text
     * @return true if the query matches with the given text
     */
    private final boolean matchesText(final String text) {
        boolean ret = false;
        QueryGoal.NormalizedWords words = new QueryGoal.NormalizedWords(Tokenizer.getWords(text, null).keySet());
        if (!SetTools.anymatchByTest(this.queryGoal.getExcludeWords(), words)) {
            ret = SetTools.totalInclusion(this.queryGoal.getIncludeWords(), words);
        }
        return ret;
    }
    
    protected static final boolean anymatch(final String text, final Iterator<String> keywords) {
        if (keywords == null || !keywords.hasNext()) return false;
        final SortedSet<String> textwords = (SortedSet<String>) Tokenizer.getWords(text, null).keySet();
        return SetTools.anymatchByTest(keywords, textwords);
    }

    public SolrQuery solrQuery(final ContentDomain cd, final boolean strictContentDom, final boolean getFacets, final boolean excludeintext_image) {
        if (cd == ContentDomain.IMAGE) {
        	return solrImageQuery(getFacets, strictContentDom);
        }
        final List<String> filterQueries;
		switch (cd) {
		case AUDIO:
			filterQueries = this.queryGoal.collectionAudioFilterQuery(strictContentDom);
			break;
		case VIDEO:
			filterQueries = this.queryGoal.collectionVideoFilterQuery(strictContentDom);
			break;
		case APP:
			filterQueries = this.queryGoal.collectionApplicationFilterQuery(strictContentDom);
			break;
		default:
			filterQueries = this.queryGoal.collectionTextFilterQuery(excludeintext_image);
			break;
		}
        return solrQuery(getFacets, filterQueries);
    }
    
    /**
     * @param getFacets when true, generate facets for fiels given in this.facetfields
     * @param filterQueries a mutable list of filter queries, initialized with filters related to content domain. Must not be null.
     * @return a Solr query instance ready to use
     */
    private SolrQuery solrQuery(final boolean getFacets, final List<String> filterQueries) {
        if (this.cachedQuery != null) {
            this.cachedQuery.setStart(this.offset);
            if (!getFacets) this.cachedQuery.setFacet(false);
            return this.cachedQuery;
        }
        
        // construct query
        final SolrQuery params = getBasicParams(getFacets, filterQueries);
        int rankingProfile = this.ranking.coeff_date == RankingProfile.COEFF_MAX ? 1 : (this.modifier.sitehash != null || this.modifier.sitehost != null) ? 2 : 0;
        params.setQuery(this.queryGoal.collectionTextQuery().toString());
        if (this.queryGoal.getIncludeSize() > 1) {
            params.setParam(DisMaxParams.MM, "1");
        }
        Ranking actRanking = indexSegment.fulltext().getDefaultConfiguration().getRanking(rankingProfile); // for a by-date ranking select different ranking profile

        String fq = actRanking.getFilterQuery();
        String bq = actRanking.getBoostQuery();
        String bf = actRanking.getBoostFunction();
        final String qf = actRanking.getQueryFields();
        if (!qf.isEmpty()) params.setParam(DisMaxParams.QF, qf);
        if (this.queryGoal.getIncludeSize() > 1) {
            // add boost on combined words
            if (bq.length() > 0) bq += "\n";
            bq += CollectionSchema.text_t.getSolrFieldName() + ":\"" + this.queryGoal.getIncludeString() + "\"^10";
        }
        if (fq.length() > 0) {
            String[] oldfq = params.getFilterQueries();
            ArrayList<String> newfq = new ArrayList<>(oldfq.length + 1);
            for (String x: oldfq) newfq.add(x);
            newfq.add(fq);
            params.setFilterQueries(newfq.toArray(new String[newfq.size()]));
        }
        if (bq.length() > 0) params.setParam(DisMaxParams.BQ, bq.split("[\\r\\n]+")); // split on any sequence consisting of CR and/or LF
        if (bf.length() > 0) params.setParam("boost", bf); // a boost function extension, see http://wiki.apache.org/solr/ExtendedDisMax#bf_.28Boost_Function.2C_additive.29
        
        // set highlighting query attributes
        if (!this.snippetFetchFullText && (this.contentdom == Classification.ContentDomain.TEXT || this.contentdom == Classification.ContentDomain.ALL)) {
        	params.setHighlight(true);
        	params.setHighlightFragsize(SearchEvent.SNIPPET_MAX_LENGTH);
            //params.setHighlightRequireFieldMatch();
        	params.setHighlightSimplePost("</b>");
        	params.setHighlightSimplePre("<b>");
        	params.setHighlightSnippets(5);
            for (final CollectionSchema field: SOLR_SNIPPET_FIELDS) {
            	params.addHighlightField(field.getSolrFieldName());
            }
        } else {
            params.setHighlight(false);
        }
        
        // prepare result
        ConcurrentLog.info("Protocol", "SOLR QUERY: " + params.toString());
        this.cachedQuery = params;
        return params;
    }
    
    private SolrQuery solrImageQuery(final boolean getFacets, final boolean strictContentDom) {
        if (this.cachedQuery != null) {
            this.cachedQuery.setStart(this.offset);
            if (!getFacets) this.cachedQuery.setFacet(false);
            return this.cachedQuery;
        }
        
        // construct query
        final SolrQuery params = getBasicParams(getFacets, this.queryGoal.collectionImageFilterQuery(strictContentDom));
        params.setQuery(this.queryGoal.collectionImageQuery(this.modifier).toString());
        
        if(!strictContentDom) {
        	// set boosts
        	StringBuilder bq = new StringBuilder();
        	bq.append(CollectionSchema.url_file_ext_s.getSolrFieldName()).append(":\"jpg\"");
        	bq.append(" OR ").append(CollectionSchema.url_file_ext_s.getSolrFieldName()).append(":\"tif\"");
        	bq.append(" OR ").append(CollectionSchema.url_file_ext_s.getSolrFieldName()).append(":\"tiff\"");
        	bq.append(" OR ").append(CollectionSchema.url_file_ext_s.getSolrFieldName()).append(":\"png\"");
        	params.setParam(DisMaxParams.BQ, bq.toString());
        }
        
        // prepare result
        ConcurrentLog.info("Protocol", "SOLR QUERY: " + params.toString());
        this.cachedQuery = params;
        return params;
    }
    
	/**
	 * Fill the Solr parameters with the relevant values to apply the search
	 * navigator sort properties.
	 * 
	 * @param params  the Solr parameters to modify
	 * @param a       Solr field name
	 * @param navSort navigator sort properties to apply
	 */
	private void fillSolrParamWithNavSort(final SolrQuery params, final String solrFieldName,
			final NavigatorSort navSort) {
		if (params != null && solrFieldName != null && navSort != null) {
			switch (navSort) {
			case COUNT_ASC:
				params.setParam("f." + solrFieldName + ".facet.sort", FacetParams.FACET_SORT_COUNT);
				/*
				 * Ascending count is not supported with Solr 6.6 (but should be on Solr 7 JSON
				 * Facet API https://lucene.apache.org/solr/guide/7_6/json-facet-api.html) So we
				 * use a here a high limit and ascending resorting will be done by YaCy
				 * Navigator
				 */
				params.setParam("f." + solrFieldName + ".facet.limit",
						String.valueOf(FACETS_MAXCOUNT_FOR_RESORT_ON_SEARCH_NAV));
				break;
			case LABEL_DESC:
				/*
				 * Descending index order is not supported with Solr 6.6 (but should be on Solr
				 * 7 JSON Facet API
				 * https://lucene.apache.org/solr/guide/7_6/json-facet-api.html) So we use a
				 * here a high limit and descending resorting will be done by YaCy Navigator
				 */
				params.setParam("f." + solrFieldName + ".facet.sort", FacetParams.FACET_SORT_INDEX);
				params.setParam("f." + solrFieldName + ".facet.limit",
						String.valueOf(FACETS_MAXCOUNT_FOR_RESORT_ON_SEARCH_NAV));
				break;
			case LABEL_ASC:
				params.setParam("f." + solrFieldName + ".facet.sort", FacetParams.FACET_SORT_INDEX);
				break;
			default:
				/* Nothing to add for COUNT_DESC which is the default for Solr */
				break;
			}

			if (CollectionSchema.language_s.getSolrFieldName().equals(solrFieldName)
					|| CollectionSchema.url_file_ext_s.getSolrFieldName().equals(solrFieldName)
					|| CollectionSchema.collection_sxt.getSolrFieldName().equals(solrFieldName)) {
				/*
				 * For these search navigators additional filtering or resorting is done in the navigator itself. 
				 * So we use a here a high limit so that the navigator apply its rules without missing elements.
				 */
				params.setParam("f." + solrFieldName + ".facet.limit",
						String.valueOf(FACETS_MAXCOUNT_FOR_RESORT_ON_SEARCH_NAV));
			}
		}
	}
    
    private SolrQuery getBasicParams(final boolean getFacets, final List<String> fqs) {
        final SolrQuery params = new SolrQuery();
        params.setParam("defType", "edismax");
        params.setParam(DisMaxParams.QF, CollectionSchema.text_t.getSolrFieldName() + "^1.0");
        params.setStart(this.offset);
        params.setRows(this.itemsPerPage);
        params.setFacet(false);

        if (this.ranking.coeff_date == RankingProfile.COEFF_MAX) {
            // set a most-recent ordering
            params.setSort(new SortClause(CollectionSchema.last_modified.getSolrFieldName(), SolrQuery.ORDER.desc));
            //params.setSortField(CollectionSchema.last_modified.getSolrFieldName(), ORDER.desc); // deprecated in Solr 4.2
        }
        
        // add site facets
        fqs.addAll(getFacetsFilterQueries());
        if (fqs.size() > 0) {
            params.setFilterQueries(fqs.toArray(new String[fqs.size()]));
        }
        
        // set facet query attributes
        if (getFacets && this.facetfields.size() > 0) {
            params.setFacet(true);
            params.setFacetMinCount(1);
            params.setFacetLimit(this.standardFacetsMaxCount);
            params.setFacetSort(FacetParams.FACET_SORT_COUNT);
            params.setParam(FacetParams.FACET_METHOD, FacetParams.FACET_METHOD_enum); // fight the fieldcache
            for (final Entry<String, NavigatorSort> entry : this.facetfields.entrySet()) {
            	params.addFacetField("{!ex=" + entry.getKey() + "}" + entry.getKey()); // params.addFacetField("{!ex=" + field + "}" + field);
                fillSolrParamWithNavSort(params, entry.getKey(), entry.getValue());                	
            }
            final NavigatorSort datesInContentSort = this.facetfields.get(CollectionSchema.dates_in_content_dts.name());
            if (datesInContentSort != null) {
            	params.setParam(FacetParams.FACET_RANGE, CollectionSchema.dates_in_content_dts.name());
                String start = new Date(System.currentTimeMillis() - 1000L * 60L * 60L * 24L * 3).toInstant().toString();
                String end = new Date(System.currentTimeMillis() + 1000L * 60L * 60L * 24L * 3).toInstant().toString();
                params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.start", start);
                params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.end", end);
                params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.range.gap", "+1DAY");
                fillSolrParamWithNavSort(params, CollectionSchema.dates_in_content_dts.getSolrFieldName(), datesInContentSort);
                params.setParam("f." + CollectionSchema.dates_in_content_dts.getSolrFieldName() + ".facet.limit", Integer.toString(this.dateFacetMaxCount)); // the year constraint should cause that limitation already
            }
            //for (String k: params.getParameterNames()) {ArrayList<String> al = new ArrayList<>(); for (String s: params.getParams(k)) al.add(s); System.out.println("Parameter: " + k + "=" + al.toString());}
            //http://localhost:8090/solr/collection1/select?q=*:*&rows=0&facet=true&facet.field=dates_in_content_dts&f.dates_in_content_dts.facet.limit=730&f.dates_in_content_dts.facet.sort=index
        } else {
            params.setFacet(false);
        }
        params.setFields(buildResultFieldList(this.snippetFetchFullText));
        return params;
    }

    private String[] buildResultFieldList(final boolean includeTextField) {
        final ArrayList<String> fields = new ArrayList<>();
        for (final CollectionSchema field : CollectionSchema.values()) {
            if (!this.solrSchema.contains(field) || !field.isStored()) {
                continue;
            }
            if (!includeTextField && field == CollectionSchema.text_t) {
                continue;
            }
            fields.add(field.getSolrFieldName());
        }
        fields.add("score");
        return fields.toArray(new String[fields.size()]);
    }
    
    long year = 1000L * 60L * 60L * 24L * 365L;
    
    private List<String> getFacetsFilterQueries() {
        
        ArrayList<String> fqs = new ArrayList<>();
        
        // add site facets
        if (this.modifier.sitehash == null && this.modifier.sitehost == null) {
            if (this.siteexcludes != null) {
                for (String ex: this.siteexcludes) {
                    fqs.add("-" + CollectionSchema.host_id_s.getSolrFieldName() + ':' + ex);
                }
            }
        } else {
            if (this.modifier.sitehost != null) {
                // consider to search for hosts with 'www'-prefix, if not already part of the host name
                if (this.modifier.sitehost.startsWith("www.")) {
                    fqs.add(CollectionSchema.host_s.getSolrFieldName() + ":\"" + this.modifier.sitehost.substring(4) + "\" OR " + CollectionSchema.host_s.getSolrFieldName() + ":\"" + this.modifier.sitehost + "\"");
                } else {
                    fqs.add(CollectionSchema.host_s.getSolrFieldName() + ":\"" + this.modifier.sitehost + "\" OR " + CollectionSchema.host_s.getSolrFieldName() + ":\"www." + this.modifier.sitehost + "\"");
                }
            } else
                fqs.add(CollectionSchema.host_id_s.getSolrFieldName() + ":\"" + this.modifier.sitehash + '\"');
        }

        // add vocabulary facets
        if (this.metatags != null) {
            for (Tagging.Metatag tag : this.metatags) {
                fqs.add(CollectionSchema.VOCABULARY_PREFIX + tag.getVocabularyName() + CollectionSchema.VOCABULARY_TERMS_SUFFIX + ":\"" + tag.getObject() + '\"');
            }
        }

        // add language facet
        if (this.modifier.language != null && this.modifier.language.length() > 0 && this.solrSchema.contains((CollectionSchema.language_s))) {
            fqs.add(CollectionSchema.language_s.getSolrFieldName() + ":\"" + this.modifier.language + '\"');
        }

        // add author facets (check for contains(author) as author_sxt is omitted copyfield)
        if (this.modifier.author != null && this.modifier.author.length() > 0 && this.solrSchema.contains(CollectionSchema.author)) {
            fqs.add(CollectionSchema.author_sxt.getSolrFieldName() + ":\"" + this.modifier.author + '\"');
        }

        // add keyword filter
        if (this.modifier.keyword != null && this.modifier.keyword.length() > 0 && this.solrSchema.contains(CollectionSchema.keywords)) {
            fqs.add(CollectionSchema.keywords.getSolrFieldName() + ":\"" + this.modifier.keyword + '\"');
        }

        // add collection facets
        if (this.modifier.collection != null && this.modifier.collection.length() > 0 && this.solrSchema.contains(CollectionSchema.collection_sxt)) {
            fqs.add(QueryModifier.parseCollectionExpression(this.modifier.collection));
        }
        
        if (this.solrSchema.contains(CollectionSchema.dates_in_content_dts)) {
            if (this.modifier.on != null && this.modifier.on.length() > 0) {
                fqs.add(QueryModifier.parseOnExpression(this.modifier.on, this.timezoneOffset));
            }
            
            if (this.modifier.from != null && this.modifier.from.length() > 0 && (this.modifier.to == null || this.modifier.to.equals("*"))) {
                fqs.add(QueryModifier.parseFromToExpression(this.modifier.from, null, this.timezoneOffset));
            }
            
            if ((this.modifier.from == null || this.modifier.from.equals("*")) && this.modifier.to != null && this.modifier.to.length() > 0) {
                fqs.add(QueryModifier.parseFromToExpression(null, this.modifier.to, this.timezoneOffset));
            }
            
            if (this.modifier.from != null && this.modifier.from.length() > 0 && this.modifier.to != null && this.modifier.to.length() > 0) {
                fqs.add(QueryModifier.parseFromToExpression(this.modifier.from, this.modifier.to, this.timezoneOffset));
            }
        }
        
        if (this.modifier.protocol != null) {
            fqs.add("{!tag=" + CollectionSchema.url_protocol_s.getSolrFieldName() + "}" + CollectionSchema.url_protocol_s.getSolrFieldName() + ':' + this.modifier.protocol);
        }
        
        if (this.tld != null) {
        	/* Use the host_s field which is mandatory, rather than the optional host_dnc_s field */
            fqs.add(CollectionSchema.host_s.getSolrFieldName() + ":*." + this.tld);
        }
        
        if (this.modifier.filetype != null) {
            fqs.add(CollectionSchema.url_file_ext_s.getSolrFieldName() + ":\"" + this.modifier.filetype + '\"');
        }
        
        if (this.inlink != null) {
            fqs.add(CollectionSchema.outboundlinks_urlstub_sxt.getSolrFieldName() + ":\"" + this.inlink + '\"');
        }
        
        if (!this.urlMask_isCatchall && this.urlMaskPattern != null) {
            // add a filter query on urls only if user custom and not generated from other modifiers
            fqs.add(CollectionSchema.sku.getSolrFieldName() + ":/" + this.urlMaskString + "/");
        }
        
        if (this.radius > 0.0d && this.lat != 0.0d && this.lon != 0.0d) {
            // localtion search, no special ranking
            // try http://localhost:8090/solr/select?q=*:*&fq={!bbox sfield=coordinate_p pt=50.17,8.65 d=1}

            //params.setQuery("!bbox " + q.toString());
            //params.set("sfield", YaCySchema.coordinate_p.name());
            //params.set("pt", Double.toString(this.lat) + "," + Double.toString(this.lon));
            //params.set("d", GeoLocation.degreeToKm(this.radius));
            fqs.add("{!bbox sfield=" + CollectionSchema.coordinate_p.getSolrFieldName() + " pt=" + Double.toString(this.lat) + "," + Double.toString(this.lon) + " d=" + GeoLocation.degreeToKm(this.radius) + "}");
            //params.setRows(Integer.MAX_VALUE);
        }
        
        return fqs;
    }
    
    public QueryGoal getQueryGoal() {
        return this.queryGoal;
    }

    public final Map<AnchorURL, String> separateMatches(final Map<AnchorURL, String> links) {
        final Map<AnchorURL, String> matcher = new HashMap<>();
        final Iterator <Map.Entry<AnchorURL, String>> i = links.entrySet().iterator();
        Map.Entry<AnchorURL, String> entry;
        AnchorURL url;
        String anchorText;
        while (i.hasNext()) {
            entry = i.next();
            url = entry.getKey();
            anchorText = entry.getValue();
            if (matchesText(anchorText)) {
                matcher.put(url, anchorText);
                i.remove();
            }
        }
        return matcher;
    }

    private volatile String idCacheAnon = null, idCache = null;
    final static private char asterisk = '*';
    public String id(final boolean anonymized) {
        if (anonymized) {
            if (this.idCacheAnon != null) return this.idCacheAnon;
        } else {
            if (this.idCache != null) return this.idCache;
        }
        synchronized (this) {
            // do a Double-Checked Locking
            if (anonymized) {
                if (this.idCacheAnon != null) return this.idCacheAnon;
            } else {
                if (this.idCache != null) return this.idCache;
            }
            // generate a string that identifies a search so results can be re-used in a cache
            final StringBuilder context = new StringBuilder(180);
            if (anonymized) {
                context.append(anonymizedQueryHashes(this.queryGoal.getIncludeHashes()));
                context.append('-');
                context.append(anonymizedQueryHashes(this.queryGoal.getExcludeHashes()));
            } else {
                context.append(hashSet2hashString(this.queryGoal.getIncludeHashes()));
                context.append('-');
                context.append(hashSet2hashString(this.queryGoal.getExcludeHashes()));
            }
            //context.append(asterisk);
            //context.append(this.domType);
            context.append(asterisk);
            context.append(this.contentdom).append(asterisk);
            context.append(this.strictContentDom).append(asterisk);
            context.append(this.zonecode).append(asterisk);
            context.append(ASCII.String(Word.word2hash(this.ranking.toExternalString()))).append(asterisk);
            context.append(Base64Order.enhancedCoder.encodeString(this.prefer.toString())).append(asterisk);
            context.append(Base64Order.enhancedCoder.encodeString(this.urlMaskString)).append(asterisk);
            context.append(this.modifier.sitehash).append(asterisk);
            context.append(this.modifier.author).append(asterisk);
            context.append(this.modifier.protocol).append(asterisk);
            context.append(this.modifier.filetype).append(asterisk);
            context.append(this.modifier.collection).append(asterisk);
            context.append(this.modifier.toString()).append(asterisk);
            context.append(this.siteexcludes).append(asterisk);
            context.append(this.targetlang).append(asterisk);
            context.append(this.domType).append(asterisk);
            context.append(this.constraint).append(asterisk);
            context.append(this.maxDistance).append(asterisk);
            context.append(this.tld).append(asterisk);
            context.append(this.inlink).append(asterisk);
            context.append(this.lat).append(asterisk).append(this.lon).append(asterisk).append(this.radius).append(asterisk);
            context.append(this.snippetCacheStrategy == null ? "null" : this.snippetCacheStrategy.name()).append(asterisk);
            context.append(this.snippetFetchFullText).append(asterisk);
            context.append(this.localSolrOnly);
            
            // Note : this.maxSuggestions search parameter do not need to be part of this id, as it has no impact on results themselves
            
            String result = context.toString();
            if (anonymized) {
                this.idCacheAnon = result;
            } else {
                this.idCache = result;
            }
            return result;
        }
    }

    public boolean isSnippetFetchFullText() {
        return this.snippetFetchFullText;
    }

    public void setSnippetFetchFullText(final boolean snippetFetchFullText) {
        this.snippetFetchFullText = snippetFetchFullText;
    }

    public boolean isLocalSolrOnly() {
        return this.localSolrOnly;
    }

    public void setLocalSolrOnly(final boolean localSolrOnly) {
        this.localSolrOnly = localSolrOnly;
    }

    /**
	 * Build a search query URL from the given parameters.
	 * 
	 * @param ext extension of the servlet to request (e.g. "html", "rss", "json"...)
	 * @param page index of the wanted page (first page is zero)
	 * @param theQuery holds the main query parameters. Must not be null.
	 * @param newModifier a eventual new modifier to append to the eventual ones already defined in theQuery QueryParams. Can be null.
	 * @param newModifierReplacesOld when newModifier is not null, it is appended in addition
	 *            to existing modifier(s) - if it is empty it overwrites (clears) existing
	 *            modifier(s)
	 * @param authenticatedFeatures
	 *            when true, access to authentication protected search features is
	 *            wanted
	 * @return a StringBuilder instance with the URL to the new search result page
	 */
	public static StringBuilder navurl(final RequestHeader.FileType ext, final int page, final QueryParams theQuery,
			final String newModifier, boolean newModifierReplacesOld, final boolean authenticatedFeatures) {

		final StringBuilder sb = navurlBase(ext, theQuery, newModifier, newModifierReplacesOld,
				authenticatedFeatures);

        sb.append("&startRecord=");
        sb.append(page * theQuery.itemsPerPage());

        return sb;
    }
	
    /**
	 * Build a search query URL from the given parameters, removing only the given single query modifier.
	 * 
	 * @param ext extension of the servlet to request (e.g. "html", "rss", "json"...)
	 * @param page index of the wanted page (first page is zero)
	 * @param theQuery holds the main query parameters. Must not be null.
	 * @param modifierToRemove the query modifier to remove (e.g. "keyword:word", "/language/en", "site:example.org"...)
	 * @param authenticatedFeatures
	 *            when true, access to authentication protected search features is
	 *            wanted
	 * @return the URL to the new search result page
	 */
	public static String navUrlWithSingleModifierRemoved(final RequestHeader.FileType ext, final int page, final QueryParams theQuery,
			final String modifierToRemove, final boolean authenticatedFeatures) {

        final StringBuilder sb = new StringBuilder(120);
        sb.append("yacysearch.");
        sb.append(ext.name().toLowerCase(Locale.ROOT));
        sb.append("?query=");

        sb.append(theQuery.getQueryGoal().getQueryString(true));
        
        if (!theQuery.modifier.isEmpty()) {
        	String modifierString = theQuery.modifier.toString();
        	if(StringUtils.isNotBlank(modifierToRemove)) {
        		if(modifierString.startsWith(modifierToRemove)) {
        			modifierString = modifierString.substring(modifierToRemove.length());
        		} else {
        			modifierString = modifierString.replace(" " + modifierToRemove, "");
        		}
        	}
        	if(StringUtils.isNotBlank(modifierString)) {
        		sb.append("+" + modifierString.trim());
        	}
        }
        
        appendNavUrlQueryParams(sb, theQuery, authenticatedFeatures);

        return sb.toString();
    }
	
    /**
	 * Build a search query URL with a new search query string, but keeping any already defined eventual modifiers.
	 * 
	 * @param ext extension of the servlet to request (e.g. "html", "rss", "json"...)
	 * @param page index of the wanted page (first page is zero)
	 * @param theQuery holds the main query parameters. Must not be null.
	 * @param authenticatedFeatures
	 *            when true, access to authentication protected search features is
	 *            wanted
	 * @return the URL to the new search result page
	 */
	public static String navUrlWithNewQueryString(final RequestHeader.FileType ext, final int page, final QueryParams theQuery,
			final String newQueryString, final boolean authenticatedFeatures) {

        final StringBuilder sb = new StringBuilder(120);
        sb.append("yacysearch.");
        sb.append(ext.name().toLowerCase(Locale.ROOT));
        sb.append("?query=");

        sb.append(new QueryGoal(newQueryString).getQueryString(true));
        
        if (!theQuery.modifier.isEmpty()) {
        	sb.append("+" + theQuery.modifier.toString());
        }
        
        appendNavUrlQueryParams(sb, theQuery, authenticatedFeatures);

        return sb.toString();
    }

     /**
	 * construct navigator url
	 *
	 * @param ext
	 *            extension of servlet (e.g. html, rss)
	 * @param theQuery
	 *            search query
	 * @param newModifier optional new modifier. - if null existing modifier(s) of theQuery are
	 *            appended - if not null this new modifier is appended in addition
	 *            to eventually existing modifier(s) - if isEmpty overwrites (clears) any eventual existing
	 *            modifier(s)
	 * @param newModifierReplacesOld considered only when newModifier is not null and not empty. When true, any existing modifiers with the same name are replaced with the new one.
	 * @param authenticatedFeatures
	 *            when true, access to authentication protected search features is
	 *            wanted
	 * @return url to new search result page
	 */
	public static StringBuilder navurlBase(final RequestHeader.FileType ext, final QueryParams theQuery,
			final String newModifier, final boolean newModifierReplacesOld, final boolean authenticatedFeatures) {

        final StringBuilder sb = new StringBuilder(120);
        sb.append("yacysearch.");
        sb.append(ext.name().toLowerCase(Locale.ROOT));
        sb.append("?query=");

        sb.append(theQuery.getQueryGoal().getQueryString(true));
        
		if (newModifier == null) {
            if (!theQuery.modifier.isEmpty()) {
            	sb.append("+" + theQuery.modifier.toString());
            }
        } else {
            if (!newModifier.isEmpty()) {
                if (!theQuery.modifier.isEmpty()) {
                	sb.append("+" + theQuery.modifier.toString());
                }
                if (newModifierReplacesOld) {
                    removeOldModifiersFromNavUrl(sb, newModifier);
                }
                try {
                	sb.append("+" + URLEncoder.encode(newModifier, StandardCharsets.UTF_8.name()));
                } catch (final UnsupportedEncodingException e) {
                	sb.append("+" + newModifier);
                }
            }
        }
		
        appendNavUrlQueryParams(sb, theQuery, authenticatedFeatures);

        return sb;
    }

    /**
	 * Append search query parameters to the URL builder already filled with the beginning of the URL.
	 * 
	 * @param sb the URL string builder to fill. Must not be null.
	 * @param theQuery holds the main query parameters. Must not be null.
	 * @param authenticatedFeatures
	 *            when true, access to authentication protected search features is
	 *            wanted
	 */
	protected static void appendNavUrlQueryParams(final StringBuilder sb, final QueryParams theQuery,
			final boolean authenticatedFeatures) {
        sb.append("&maximumRecords=");
        sb.append(theQuery.itemsPerPage());

        sb.append("&resource=");
        sb.append((theQuery.isLocal()) ? "local" : "global");

        sb.append("&verify=");
        sb.append(theQuery.snippetCacheStrategy == null ? "false" : theQuery.snippetCacheStrategy.toName());

        sb.append("&prefermaskfilter=");
        sb.append(theQuery.prefer);

        sb.append("&cat=href");

        sb.append("&constraint=");
        sb.append((theQuery.constraint == null) ? "" : theQuery.constraint.exportB64());

        sb.append("&contentdom=");
        sb.append(theQuery.contentdom.toString());
        
        sb.append("&strictContentDom=");
        sb.append(String.valueOf(theQuery.isStrictContentDom()));
        
        sb.append("&meanCount=");
        sb.append(theQuery.getMaxSuggestions());

        sb.append("&former=");
        sb.append(theQuery.getQueryGoal().getQueryString(true));

        if(authenticatedFeatures) {
        	sb.append("&auth");
        }
	}

	/**
	 * Remove from the URL builder any query modifiers with the same name that the new modifier 
	 * @param sb
	 *            a StringBuilder holding the search URL navigation being built.
	 *            Must not be null and contain the URL base and the query string
	 *            with its eventual modifiers
	 * @param newModifier
	 *            a new modifier of form key:value. Must not be null.
	 */
	protected static void removeOldModifiersFromNavUrl(final StringBuilder sb, final String newModifier) {
		int nmpi = newModifier.indexOf(":");
		if (nmpi > 0) {
		    final String newModifierKey = newModifier.substring(0, nmpi) + ":";
		    int sameModifierIndex = sb.indexOf(newModifierKey);
		    while (sameModifierIndex > 0) {
		    	final int spaceModifierIndex = sb.indexOf(" ", sameModifierIndex);
		    	if(spaceModifierIndex > sameModifierIndex) {
		    		/* There are other modifiers after the matching one : we only remove the old matching modifier */
		    		sb.delete(sameModifierIndex, spaceModifierIndex + 1);
		    	} else {
		    		/* The matching modifier is the last : we truncate the builder */
		        	sb.setLength(sameModifierIndex);	
		    	}
		    	sameModifierIndex = sb.indexOf(newModifierKey);
		    }
		    if (sb.charAt(sb.length() - 1) == '+') {
		    	sb.setLength(sb.length() - 1);
		    }
		    if (sb.charAt(sb.length() - 1) == ' ') {
		    	sb.setLength(sb.length() - 1);
		    }
		}
	}

}
