/**
 *  TextParser.java
 *  Copyright 2009 by Michael Peter Christen, mc@yacy.net, Frankfurt am Main, Germany
 *  First released 09.07.2009 at https://yacy.net
 * <p>
 *  This library is free software; you can redistribute it and/or
 *  modify it under the terms of the GNU Lesser General Public
 *  License as published by the Free Software Foundation; either
 *  version 2.1 of the License, or (at your option) any later version.
 * <p>
 *  This library is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 *  Lesser General Public License for more details.
 * <p>
 *  You should have received a copy of the GNU Lesser General Public License
 *  along with this program in the file lgpl21.txt
 *  If not, see <http://www.gnu.org/licenses/>.
 */

package net.yacy.document;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

import org.apache.commons.io.input.CloseShieldInputStream;

import net.yacy.cora.document.encoding.UTF8;
import net.yacy.cora.document.id.DigestURL;
import net.yacy.cora.document.id.MultiProtocolURL;
import net.yacy.cora.util.CommonPattern;
import net.yacy.cora.util.StrictLimitInputStream;
import net.yacy.document.parser.GenericXMLParser;
import net.yacy.document.parser.XZParser;
import net.yacy.document.parser.apkParser;
import net.yacy.document.parser.audioTagParser;
import net.yacy.document.parser.bzipParser;
import net.yacy.document.parser.csvParser;
import net.yacy.document.parser.docParser;
import net.yacy.document.parser.genericParser;
import net.yacy.document.parser.gzipParser;
import net.yacy.document.parser.gzipParser.GZIPOpeningStreamException;
import net.yacy.document.parser.htmlParser;
import net.yacy.document.parser.linkScraperParser;
import net.yacy.document.parser.mmParser;
import net.yacy.document.parser.odtParser;
import net.yacy.document.parser.ooxmlParser;
import net.yacy.document.parser.pdfParser;
import net.yacy.document.parser.pptParser;
import net.yacy.document.parser.psParser;
import net.yacy.document.parser.rssParser;
import net.yacy.document.parser.rtfParser;
import net.yacy.document.parser.sidAudioParser;
import net.yacy.document.parser.tarParser;
import net.yacy.document.parser.torrentParser;
import net.yacy.document.parser.vcfParser;
import net.yacy.document.parser.vsdParser;
import net.yacy.document.parser.xlsParser;
import net.yacy.document.parser.zipParser;
import net.yacy.document.parser.html.TagValency;
import net.yacy.document.parser.images.genericImageParser;
import net.yacy.document.parser.images.metadataImageParser;
import net.yacy.document.parser.images.svgParser;
import net.yacy.kelondro.util.FileUtils;
import net.yacy.kelondro.util.MemoryControl;

import java.util.*;

public final class TextParser {

    private static final Object v = new Object();

    private static final Parser genericIdiom = new genericParser();

    /** A generic XML parser instance */
    private static final Parser genericXMLIdiom = new GenericXMLParser();

    //use LinkedHashSet for parser collection to use (init) order to prefered parser for same ext or mime
    private static final Map<String, LinkedHashSet<Parser>> mime2parser = new ConcurrentHashMap<>();
    private static final ConcurrentHashMap<String, LinkedHashSet<Parser>> ext2parser = new ConcurrentHashMap<>();
    private static final Map<String, String> ext2mime = new ConcurrentHashMap<>();
    private static final Map<String, Object> denyMime = new ConcurrentHashMap<>();
    private static final Map<String, Object> denyExtensionx = new ConcurrentHashMap<>();

    static {
        initParser(new apkParser());
        initParser(new bzipParser());
        initParser(new XZParser());
        initParser(new csvParser());
        initParser(new docParser());
        initParser(new gzipParser());
        // AugmentParser calls internally RDFaParser (therefore add before RDFa)
        // if (Switchboard.getSwitchboard().getConfigBool("parserAugmentation", true)) initParser(new AugmentParser()); // experimental implementation, not working yet (2015-06-05)
        // RDFaParser calls internally htmlParser (therefore add before html)
        // if (Switchboard.getSwitchboard().getConfigBool("parserAugmentation.RDFa", true)) initParser(new RDFaParser()); // experimental implementation, not working yet (2015-06-04)
        initParser(new htmlParser()); // called within rdfa parser
        initParser(new genericImageParser());
        initParser(new metadataImageParser());
        initParser(new linkScraperParser());
        initParser(new mmParser());
        initParser(new odtParser());
        initParser(new ooxmlParser());
        initParser(new pdfParser());
        initParser(new pptParser());
        initParser(new psParser());
        initParser(new rssParser());
        initParser(new rtfParser());
        initParser(new sidAudioParser());
        initParser(new svgParser());
        initParser(new tarParser());
        initParser(new torrentParser());
        initParser(new vcfParser());
        initParser(new vsdParser());
        initParser(new xlsParser());
        initParser(new zipParser());
        initParser(new audioTagParser());
        /* Order is important : the generic XML parser must be initialized in last, so it will be effectively used only as a fallback one
         * when a specialized parser exists for any XML based format (examples : rssParser or ooxmlParser must be tried first) */
        initParser(genericXMLIdiom);
    }

    public static Set<Parser> parsers() {
        final Set<Parser> c = new HashSet<>();
        for (final Set<Parser> pl: ext2parser.values()) c.addAll(pl);
        for (final Set<Parser> pl: mime2parser.values()) c.addAll(pl);
        return c;
    }

    /**
     * @return the set of all supported mime types
     */
    public static Set<String> supportedMimeTypes() {
        final Set<String> mimeTypes = new HashSet<>();
        mimeTypes.addAll(mime2parser.keySet());
        return mimeTypes;
    }

    private static void initParser(final Parser parser) {
        String prototypeMime = null;
        for (final String mime: parser.supportedMimeTypes()) {
            // process the mime types
            final String mimeType = normalizeMimeType(mime);
            if (prototypeMime == null) prototypeMime = mimeType;
            LinkedHashSet<Parser> p0 = mime2parser.get(mimeType);
            if (p0 == null) {
                p0 = new LinkedHashSet<>();
                mime2parser.put(mimeType, p0);
            }
            p0.add(parser);
            AbstractParser.log.info("Parser for mime type '" + mimeType + "': " + parser.getName());
        }

        if (prototypeMime != null) for (String ext: parser.supportedExtensions()) {
            ext = ext.toLowerCase(Locale.ROOT);
            final String s = ext2mime.get(ext);
            if (s != null && !s.equals(prototypeMime)) AbstractParser.log.info("Parser for extension '" + ext + "' was set to mime '" + s + "', overwriting with new mime '" + prototypeMime + "'.");
            ext2mime.put(ext, prototypeMime);
        }

        for (String ext: parser.supportedExtensions()) {
            // process the extensions
            ext = ext.toLowerCase(Locale.ROOT);
            LinkedHashSet<Parser> p0 = ext2parser.get(ext);
            if (p0 == null) {
                p0 = new LinkedHashSet<>();
                ext2parser.put(ext, p0);
            }
            p0.add(parser);
            AbstractParser.log.info("Parser for extension '" + ext + "': " + parser.getName());
        }
    }

    public static Document[] parseSource(
            final DigestURL location,
            final String mimeType,
            final String charset,
            final TagValency defaultValency,
            final Set<String> valencySwitchTagNames,
            final VocabularyScraper scraper,
            final int timezoneOffset,
            final int depth,
            final File sourceFile,
            final Date lastModified
            ) throws InterruptedException, Parser.Failure {

        BufferedInputStream sourceStream = null;
        Document[] docs = null;
        try {
            if (AbstractParser.log.isFine()) AbstractParser.log.fine("Parsing '" + location + "' from file");
            if (!sourceFile.exists() || !sourceFile.canRead() || sourceFile.length() == 0) {
                final String errorMsg = sourceFile.exists() ? "Empty resource file." : "No resource content available (2).";
                AbstractParser.log.info("Unable to parse '" + location + "'. " + errorMsg);
                throw new Parser.Failure(errorMsg, location);
            }
            sourceStream = new BufferedInputStream(new FileInputStream(sourceFile));
            docs = parseSource(location, mimeType, charset, defaultValency, valencySwitchTagNames, scraper, timezoneOffset, depth, sourceFile.length(), sourceStream, lastModified);
        } catch (final Exception e) {
            if (e instanceof InterruptedException) throw (InterruptedException) e;
            if (e instanceof Parser.Failure) throw (Parser.Failure) e;
            AbstractParser.log.severe("Unexpected exception in parseSource from File: " + e.getMessage(), e);
            throw new Parser.Failure("Unexpected exception: " + e.getMessage(), location);
        } finally {
            if (sourceStream != null) try { sourceStream.close(); } catch (final Exception ex) {}
        }

        return docs;
    }

    public static Document[] parseSource(
            final DigestURL location,
            String mimeType,
            final String charset,
            final TagValency defaultValency,
            final Set<String> valencySwitchTagNames,
            final VocabularyScraper scraper,
            final int timezoneOffset,
            final int depth,
            final byte[] content,
            final Date lastModified
            ) throws Parser.Failure {
        if (AbstractParser.log.isFine()) AbstractParser.log.fine("Parsing '" + location + "' from byte-array");
        mimeType = normalizeMimeType(mimeType);
        Set<Parser> idioms = null;
        try {
            idioms = parsers(location, mimeType);
        } catch (final Parser.Failure e) {
            final String errorMsg = "Parser Failure for extension '" + MultiProtocolURL.getFileExtension(location.getFileName()) + "' or mimetype '" + mimeType + "': " + e.getMessage();
            AbstractParser.log.warn(errorMsg);
            throw new Parser.Failure(errorMsg, location);
        }
        assert !idioms.isEmpty() : "no parsers applied for url " + location.toNormalform(true);

        final Document[] docs = parseSource(location, mimeType, idioms, charset, defaultValency, valencySwitchTagNames, scraper, timezoneOffset, depth, content, Integer.MAX_VALUE, Long.MAX_VALUE, lastModified);

        return docs;
    }

    /**
     * Apply only the generic parser to the given content from location.
     */
    public static Document[] genericParseSource(
            final DigestURL location,
            String mimeType,
            final String charset,
            final TagValency defaultValency,
            final Set<String> valencySwitchTagNames,
            final VocabularyScraper scraper,
            final int timezoneOffset,
            final int depth,
            final byte[] content,
            final Date lastModified
            ) throws Parser.Failure {
        if (AbstractParser.log.isFine()) {
            AbstractParser.log.fine("Parsing '" + location + "' from byte-array, applying only the generic parser");
        }
        mimeType = normalizeMimeType(mimeType);
        final Set<Parser> idioms = new HashSet<>();
        idioms.add(TextParser.genericIdiom);

        return parseSource(location, mimeType, idioms, charset, defaultValency, valencySwitchTagNames, scraper, timezoneOffset, depth, content, Integer.MAX_VALUE, Long.MAX_VALUE, lastModified);
    }

    private static Document[] parseSource(
            final DigestURL location,
            String mimeType,
            final String charset,
            final TagValency defaultValency,
            final Set<String> valencySwitchTagNames,
            final VocabularyScraper scraper,
            final int timezoneOffset,
            final int depth,
            final long contentLength,
            final InputStream sourceStream,
            final int maxLinks,
            final long maxBytes,
            final Date lastModified
            ) throws Parser.Failure {
        if (AbstractParser.log.isFine()) AbstractParser.log.fine("Parsing '" + location + "' from stream");
        mimeType = normalizeMimeType(mimeType);
        Set<Parser> idioms = null;
        try {
            idioms = parsers(location, mimeType);
        } catch (final Parser.Failure e) {
            final String errorMsg = "Parser Failure for extension '" + MultiProtocolURL.getFileExtension(location.getFileName()) + "' or mimetype '" + mimeType + "': " + e.getMessage();
            AbstractParser.log.warn(errorMsg);
            throw new Parser.Failure(errorMsg, location);
        }
        assert !idioms.isEmpty() : "no parsers applied for url " + location.toNormalform(true);

        boolean canStream = false;
        if(idioms.size() == 1) {
            canStream = true;
        } else if(idioms.size() == 2) {
            /* When there are only 2 available parsers, stream oriented parsing can still be applied when one of the 2 parsers is the generic one */
            for(final Parser idiom : idioms) {
                if(idiom instanceof genericParser) {
                    canStream = true;
                }
            }
        } else if(sourceStream instanceof ByteArrayInputStream) {
            /* Also check if we have a ByteArrayInputStream as source to prevent useless bytes duplication in a new byte array */
            canStream = true;
        }

        // if we do not have more than one non generic parser, or the content size is over MaxInt (2GB), or is over the totally available memory,
        // or stream is already in memory as a ByteArrayInputStream
        // then we use only stream-oriented parser.
        if (canStream || contentLength > Integer.MAX_VALUE || contentLength > MemoryControl.available()) {
            try {
                /* The size of the buffer on the stream must be large enough to allow parser implementations to start parsing the resource
                 * and eventually fail, but must also be larger than eventual parsers internal buffers such as BufferedInputStream.DEFAULT_BUFFER_SIZE (8192 bytes) */
                final int rewindSize = 10 * 1024;
                final InputStream markableStream;
                if(sourceStream instanceof ByteArrayInputStream) {
                    /* No nead to use a wrapping buffered stream when the source is already entirely in memory.
                     * What's more, ByteArrayInputStream has no read limit when marking.*/
                    markableStream = sourceStream;
                } else {
                    markableStream = new BufferedInputStream(sourceStream, rewindSize);
                }
                /* Mark now to allow resetting the buffered stream to the beginning of the stream */
                markableStream.mark(rewindSize);

                /* Loop on parser : they are supposed to be sorted in order to start with the most specific and end with the most generic */
                for(final Parser parser : idioms) {
                    /* Wrap in a CloseShieldInputStream to prevent SAX parsers closing the sourceStream
                     * and so let us eventually reuse the same opened stream with other parsers on parser failure */
                    CloseShieldInputStream nonCloseInputStream = CloseShieldInputStream.wrap(markableStream);

                    try {
                        return parseSource(location, mimeType, parser, charset, defaultValency, valencySwitchTagNames, scraper, timezoneOffset,
                                nonCloseInputStream, maxLinks, maxBytes, lastModified);
                    } catch (final Parser.Failure e) {
                        /* Try to reset the marked stream. If the failed parser has consumed too many bytes :
                         * too bad, the marks is invalid and process fails now with an IOException */
                        markableStream.reset();

                        if(parser instanceof gzipParser && e.getCause() instanceof GZIPOpeningStreamException
                                && (idioms.size() == 1 || (idioms.size() == 2 && idioms.contains(genericIdiom)))) {
                            /* The gzip parser failed directly when opening the content stream : before falling back to the generic parser,
                             * let's have a chance to parse the stream as uncompressed. */
                            /* Indeed, this can be a case of misconfigured web server, providing both headers "Content-Encoding" with value "gzip",
                             * and "Content-type" with value such as "application/gzip".
                             * In that case our HTTP client (see GzipResponseInterceptor) is already uncompressing the stream on the fly,
                             * that's why the gzipparser fails opening the stream.
                             * (see RFC 7231 section 3.1.2.2 for "Content-Encoding" header specification https://tools.ietf.org/html/rfc7231#section-3.1.2.2)*/
                            final gzipParser gzParser = (gzipParser)parser;

                            nonCloseInputStream = CloseShieldInputStream.wrap(markableStream);

                            final Document maindoc = gzipParser.createMainDocument(location, mimeType, charset, gzParser);

                            try {
                                final Document[] docs = gzParser.parseCompressedInputStream(location,
                                        charset, timezoneOffset, depth,
                                        nonCloseInputStream, maxLinks, maxBytes);
                                if (docs != null) {
                                    maindoc.addSubDocuments(docs);
                                }
                                return new Document[] { maindoc };
                            } catch(final Exception e1) {
                                /* Try again to reset the marked stream if the failed parser has not consumed too many bytes */
                                markableStream.reset();
                            }
                        }
                    }
                }
            } catch (final IOException e) {
                throw new Parser.Failure("Error reading source", location);
            }
        }

        // in case that we know more parsers we first transform the content into a byte[] and use that as base
        // for a number of different parse attempts.

        int maxBytesToRead = -1;
        if(maxBytes < Integer.MAX_VALUE) {
            /* Load at most maxBytes + 1 :
               - to let parsers not supporting Parser.parseWithLimits detect the maxBytes size is exceeded and end with a Parser.Failure
               - but let parsers supporting Parser.parseWithLimits perform partial parsing of maxBytes content */
            maxBytesToRead = (int)maxBytes + 1;
        }
        if (contentLength >= 0 && contentLength < maxBytesToRead) {
            maxBytesToRead = (int)contentLength;
        }

        byte[] b = null;
        try {
            b = FileUtils.read(sourceStream, maxBytesToRead);
        } catch (final IOException e) {
            throw new Parser.Failure(e.getMessage(), location);
        }
        final Document[] docs = parseSource(location, mimeType, idioms, charset, defaultValency, valencySwitchTagNames, scraper, timezoneOffset, depth, b, maxLinks, maxBytes, lastModified);

        return docs;
    }

    public static Document[] parseSource(
            final DigestURL location,
            final String mimeType,
            final String charset,
            final TagValency defaultValency,
            final Set<String> valencySwitchTagNames,
            final VocabularyScraper scraper,
            final int timezoneOffset,
            final int depth,
            final long contentLength,
            final InputStream sourceStream,
            final Date lastModified) throws Parser.Failure {
        return parseSource(location, mimeType, charset, defaultValency, valencySwitchTagNames, scraper, timezoneOffset, depth, contentLength, sourceStream,
                Integer.MAX_VALUE, Long.MAX_VALUE, lastModified);
    }

    /**
     * Try to limit the parser processing with a maximum total number of links detection (anchors, images links, media links...)
     * or a maximum amount of content bytes to parse. Limits apply only when the available parsers for the resource media type support parsing within limits
     * (see {@link Parser#isParseWithLimitsSupported()}. When available parsers do
     * not support parsing within limits, an exception is thrown when
     * content size is beyond maxBytes.
     * @param location the URL of the source
     * @param mimeType the mime type of the source, if known
     * @param charset the charset name of the source, if known
     * @param defaultValency the valency default; should be TagValency.EVAL by default
     * @param valencySwitchTagNames the valency switch tag names
     * @param timezoneOffset the local time zone offset
     * @param depth the current depth of the crawl
     * @param contentLength the length of the source, if known (else -1 should be used)
     * @param sourceStream a input stream
     * @param maxLinks the maximum total number of links to parse and add to the result documents
     * @param maxBytes the maximum number of content bytes to process
     * @return an array of documents that result from parsing the source, with empty or null text.
     * @throws Parser.Failure when the parser processing failed
     */
    public static Document[] parseWithLimits(
            final DigestURL location,
            final String mimeType,
            final String charset,
            final TagValency defaultValency,
            final Set<String> valencySwitchTagNames,
            final int timezoneOffset,
            final int depth,
            final long contentLength,
            final InputStream sourceStream,
            final int maxLinks,
            final long maxBytes,
            final Date lastModified) throws Parser.Failure{
        return parseSource(location, mimeType, charset, defaultValency, valencySwitchTagNames, new VocabularyScraper(), timezoneOffset, depth, contentLength,
                sourceStream, maxLinks, maxBytes, lastModified);
    }

    /**
     * Try to limit the parser processing with a maximum total number of links detection (anchors, images links, media links...)
     * or a maximum amount of content bytes to parse. Limits apply only when the available parsers for the resource media type support parsing within limits
     * (see {@link Parser#isParseWithLimitsSupported()}. When available parsers do
     * not support parsing within limits, an exception is thrown when
     * content size is beyond maxBytes.
     * @param location the URL of the source
     * @param mimeType the mime type of the source, if known
     * @param charset the charset name of the source, if known
     * @param timezoneOffset the local time zone offset
     * @param depth the current depth of the crawl
     * @param contentLength the length of the source, if known (else -1 should be used)
     * @param sourceStream a input stream
     * @param maxLinks the maximum total number of links to parse and add to the result documents
     * @param maxBytes the maximum number of content bytes to process
     * @return a list of documents that result from parsing the source, with empty or null text.
     * @throws Parser.Failure when the parser processing failed
     */
    public static Document[] parseWithLimits(
            final DigestURL location, final String mimeType, final String charset,
            final int timezoneOffset, final int depth, final long contentLength, final InputStream sourceStream, final int maxLinks,
            final long maxBytes, final Date lastModified) throws Parser.Failure {
        return parseSource(location, mimeType, charset, TagValency.EVAL, new HashSet<>(), new VocabularyScraper(), timezoneOffset, depth, contentLength,
                sourceStream, maxLinks, maxBytes, lastModified);
    }

    /**
     *
     * @param location the URL of the source
     * @param mimeType the mime type of the source, if known
     * @param parser a parser supporting the resource at location
     * @param charset the charset name of the source, if known
     * @param scraper a vocabulary scraper
     * @param timezoneOffset the local time zone offset
     * @param sourceStream an open input stream on the source
     * @param maxLinks the maximum total number of links to parse and add to the result documents
     * @param maxBytes the maximum number of content bytes to process
     * @return a list of documents that result from parsing the source
     * @throws Parser.Failure when the source could not be parsed
     */
    private static Document[] parseSource(
            final DigestURL location,
            final String mimeType,
            final Parser parser,
            final String charset,
            final TagValency defaultValency,
            final Set<String> valencySwitchTagNames,
            final VocabularyScraper scraper,
            final int timezoneOffset,
            final InputStream sourceStream,
            final int maxLinks,
            final long maxBytes,
            final Date lastModified
            ) throws Parser.Failure {
        if (AbstractParser.log.isFine()) AbstractParser.log.fine("Parsing '" + location + "' from stream");
        final String fileExt = MultiProtocolURL.getFileExtension(location.getFileName());
        final String documentCharset = htmlParser.patchCharsetEncoding(charset);
        assert parser != null;

        if (AbstractParser.log.isFine()) AbstractParser.log.fine("Parsing " + location + " with mimeType '" + mimeType + "' and file extension '" + fileExt + "'.");
        try {
            final Document[] docs;
            if(parser.isParseWithLimitsSupported()) {
                docs = parser.parseWithLimits(location, mimeType, documentCharset, defaultValency, valencySwitchTagNames, scraper, timezoneOffset, sourceStream, maxLinks, maxBytes, lastModified);
            } else {
                /* Parser do not support partial parsing within limits : let's control it here*/
                final InputStream limitedSource = new StrictLimitInputStream(sourceStream, maxBytes);
                docs = parser.parse(location, mimeType, documentCharset, defaultValency, valencySwitchTagNames, scraper, timezoneOffset, limitedSource);
            }
            return docs;
        } catch(final Parser.Failure e) {
            throw e;
        } catch (final Exception e) {
            throw new Parser.Failure("parser failed: " + parser.getName(), location);
        }
    }

    /**
     * @param location the URL of the source
     * @param mimeType the mime type of the source, if known
     * @param parsers a set of parsers supporting the resource at location
     * @param charset the charset name of the source, if known
     * @param scraper a vocabulary scraper
     * @param timezoneOffset the local time zone offset
     * @param depth the current crawling depth
     * @param sourceArray the resource content bytes
     * @param maxLinks the maximum total number of links to parse and add to the result documents
     * @param maxBytes the maximum number of content bytes to process
     * @return an array of documents that result from parsing the source
     * @throws Parser.Failure when the source could not be parsed
     */
    private static Document[] parseSource(
            final DigestURL location,
            final String mimeType,
            final Set<Parser> parsers,
            final String charset,
            final TagValency defaultValency,
            final Set<String> valencySwitchTagNames,
            final VocabularyScraper scraper,
            final int timezoneOffset,
            final int depth,
            final byte[] sourceArray,
            final int maxLinks,
            final long maxBytes,
            final Date lastModified
            ) throws Parser.Failure {
        final String fileExt = MultiProtocolURL.getFileExtension(location.getFileName());
        if (AbstractParser.log.isFine()) AbstractParser.log.fine("Parsing " + location + " with mimeType '" + mimeType + "' and file extension '" + fileExt + "' from byte[]");
        final String documentCharset = htmlParser.patchCharsetEncoding(charset);
        assert !parsers.isEmpty();

        Document[] docs = null;
        final Map<Parser, Parser.Failure> failedParser = new HashMap<>();
        final String origName = Thread.currentThread().getName();
        Thread.currentThread().setName("parsing + " + location.toString()); // set a name to get the address in Thread Dump
        for (final Parser parser: parsers) {
            if (MemoryControl.request(sourceArray.length * 6, false)) {
                ByteArrayInputStream bis;
                if (mimeType.equals("text/plain") && parser.getName().equals("HTML Parser")) {
                    // a hack to simulate html files .. is needed for NOLOAD queues. This throws their data into virtual text/plain messages.
                    bis = new ByteArrayInputStream(UTF8.getBytes("<html><head></head><body><h1>" + UTF8.String(sourceArray) + "</h1></body><html>"));
                } else {
                    bis = new ByteArrayInputStream(sourceArray);
                }
                try {
                    if(parser.isParseWithLimitsSupported()) {
                        docs = parser.parseWithLimits(location, mimeType, documentCharset, defaultValency, valencySwitchTagNames, scraper, timezoneOffset, bis, maxLinks, maxBytes, lastModified);
                    } else {
                        /* Partial parsing is not supported by this parser : check content length now */
                        if(sourceArray.length > maxBytes) {
                            throw new Parser.Failure("Content size is over maximum size of " + maxBytes + "", location);
                        }
                        docs = parser.parse(location, mimeType, documentCharset, defaultValency, valencySwitchTagNames, scraper, timezoneOffset, bis);
                    }
                } catch (final Parser.Failure e) {
                    if(parser instanceof gzipParser && e.getCause() instanceof GZIPOpeningStreamException &&
                            (parsers.size() == 1 || (parsers.size() == 2 && parsers.contains(genericIdiom)))) {
                        /* The gzip parser failed directly when opening the content stream : before falling back to the generic parser,
                         * let's have a chance to parse the stream as uncompressed. */
                        /* Indeed, this can be a case of misconfigured web server, providing both headers "Content-Encoding" with value "gzip",
                         * and "Content-type" with value such as "application/gzip".
                         * In that case our HTTP client (see GzipResponseInterceptor) is already uncompressing the stream on the fly,
                         * that's why the gzipparser fails opening the stream.
                         * (see RFC 7231 section 3.1.2.2 for "Content-Encoding" header specification https://tools.ietf.org/html/rfc7231#section-3.1.2.2)*/
                        final gzipParser gzParser = (gzipParser)parser;

                        bis = new ByteArrayInputStream(sourceArray);

                        final Document maindoc = gzipParser.createMainDocument(location, mimeType, charset, gzParser);

                        try {
                            docs = gzParser.parseCompressedInputStream(location,
                                    charset, timezoneOffset, depth,
                                    bis, maxLinks, maxBytes);
                            if (docs != null) {
                                maindoc.addSubDocuments(docs);
                            }
                            docs = new Document[] { maindoc };
                            break;
                        } catch(final Parser.Failure e1) {
                            failedParser.put(parser, e1);
                        } catch(final Exception e2) {
                            failedParser.put(parser, new Parser.Failure(e2.getMessage(), location));
                        }
                    } else {
                        failedParser.put(parser, e);
                    }
                } catch (final Exception e) {
                    failedParser.put(parser, new Parser.Failure(e.getMessage(), location));
                    //log.logWarning("tried parser '" + parser.getName() + "' to parse " + location.toNormalform(true, false) + " but failed: " + e.getMessage(), e);
                } finally {
                    try {
                        bis.close();
                    } catch(final IOException ioe) {
                        // Ignore.
                    }
                }
                if (docs != null) break;
            }
        }
        Thread.currentThread().setName(origName);

        if (docs == null) {
            if (failedParser.isEmpty()) {
                final String errorMsg = "Parsing content with file extension '" + fileExt + "' and mimetype '" + mimeType + "' failed.";
                //log.logWarning("Unable to parse '" + location + "'. " + errorMsg);
                throw new Parser.Failure(errorMsg, location);
            }
            String failedParsers = "";
            for (final Map.Entry<Parser, Parser.Failure> error: failedParser.entrySet()) {
                AbstractParser.log.warn("tried parser '" + error.getKey().getName() + "' to parse " + location.toNormalform(true) + " but failed: " + error.getValue().getMessage(), error.getValue());
                failedParsers += error.getKey().getName() + " ";
            }
            throw new Parser.Failure("All parser failed: " + failedParsers, location);
        }
        for (final Document d: docs) {
            final InputStream textStream = d.getTextStream();
            assert textStream != null : "mimeType = " + mimeType;
            try {
                if(textStream != null) {
                    /* textStream can be a FileInputStream : we must close it to ensure releasing system resource */
                    textStream.close();
                }
            } catch (final IOException e) {
                AbstractParser.log.warn("Could not close text input stream");
            }
            d.setDepth(depth);
        } // verify docs

        return docs;
    }

    /**
     * check if the parser supports the given content.
     * @param url
     * @param mimeType
     * @return returns null if the content is supported. If the content is not supported, return a error string.
     */
    public static String supports(final MultiProtocolURL url, final String mimeType) {
        try {
            // try to get a parser. If this works, we don't need the parser itself, we just return null to show that everything is ok.
            final Set<Parser> idioms = parsers(url, mimeType);
            return (idioms == null || idioms.isEmpty() || (idioms.size() == 1 && idioms.iterator().next().getName().equals(genericIdiom.getName()))) ? "no parser found" : null;
        } catch (final Parser.Failure e) {
            // in case that a parser is not available, return a error string describing the problem.
            return e.getMessage();
        }
    }

    /**
     * find a parser for a given url and mime type
     * because mime types returned by web severs are sometimes wrong, we also compute the mime type again
     * from the extension that can be extracted from the url path. That means that there are 3 criteria
     * that can be used to select a parser:
     * - the given mime type (1.)
     * - the extension of url (2.)
     * - the mime type computed from the extension (3.)
     * finally the generic parser is added as backup if all above fail
     * @param url the given url
     * @param mimeType1 the given mime type
     * @return a list of Idiom parsers that may be appropriate for the given criteria
     * @throws Parser.Failure when the file extension or the MIME type is denied
     */
    private static Set<Parser> parsers(final MultiProtocolURL url, String mimeType1) throws Parser.Failure {
        final Set<Parser> idioms = new LinkedHashSet<>(2); // LinkedSet to maintain order (genericParser should be last)

        // check given mime type, place this first because this is the most likely to work and the best fit to the supplied mime
        Set<Parser> idiom;
        if (mimeType1 != null) {
            mimeType1 = normalizeMimeType(mimeType1);
            if (denyMime.containsKey(mimeType1)) throw new Parser.Failure("mime type '" + mimeType1 + "' is denied (1)", url);
            idiom = mime2parser.get(mimeType1);
            if (idiom != null) idioms.addAll(idiom);
        }

        // check extension and add as backup (in case no, wrong or unknown/unsupported mime was supplied)
        final String ext = MultiProtocolURL.getFileExtension(url.getFileName());
        if (ext != null && ext.length() > 0) {
            /* We do not throw here an exception when the media type is provided and inconsistent with the extension (if it is not supported an exception has already beeen thrown).
             * Otherwise we would reject URLs with an apparently unsupported extension but whose actual Media Type is supported (for example text/html).
             * Notable example : wikimedia commons pages, such as https://commons.wikimedia.org/wiki/File:YaCy_logo.png */
            if (denyExtensionx.containsKey(ext) && (mimeType1 == null || mimeType1.equals(mimeOf(ext)))) {
                throw new Parser.Failure("file extension '" + ext + "' is denied (1)", url);
            }
            idiom = ext2parser.get(ext);
            if (idiom != null && !idioms.containsAll(idiom)) { // use containsAll -> idiom is a Set of parser
                idioms.addAll(idiom);
            }
        }

        // check mime type computed from extension
        final String mimeType2 = ext2mime.get(ext);
        if (mimeType2 != null && (idiom = mime2parser.get(mimeType2)) != null && !idioms.containsAll(idiom)) { // use containsAll -> idiom is a Set of parser
            idioms.addAll(idiom);
        }

        /* No matching idiom has been found : let's check if the media type ends with the "+xml" suffix so we can handle it with a generic XML parser
         * (see RFC 7303 - Using '+xml' when Registering XML-Based Media Types : https://tools.ietf.org/html/rfc7303#section-4.2) */
        if(idioms.isEmpty() && mimeType1 != null && mimeType1.endsWith("+xml")) {
            idioms.add(genericXMLIdiom);
        }

        // always add the generic parser (make sure it is the last in access order)
        idioms.add(genericIdiom);
        //if (idioms.isEmpty()) throw new Parser.Failure("no parser found for extension '" + ext + "' and mime type '" + mimeType1 + "'", url);

        return idioms;
    }

    /**
     * checks if the parser supports the given mime type. It is not only checked if the parser can parse such types,
     * it is also checked if the mime type is not included in the mimetype-deny list.
     * @param mimeType
     * @return an error if the mime type is not supported, null otherwise
     */
    public static String supportsMime(String mimeType) {
        if (mimeType == null) {
            return null;
        }
        mimeType = normalizeMimeType(mimeType);
        if (denyMime.containsKey(mimeType)) {
            return "mime type '" + mimeType + "' is denied (2)";
        }
        if (mime2parser.get(mimeType) == null) {
            /* No matching idiom has been found : let's check if the media type ends with the "+xml" suffix as can handle it with a generic XML parser
             * (see RFC 7303 - Using '+xml' when Registering XML-Based Media Types : https://tools.ietf.org/html/rfc7303#section-4.2) */
            if(!mimeType.endsWith("+xml")) {
                return "no parser for mime '" + mimeType + "' available";
            }
        }
        return null;
    }

    /**
     * checks if the parser supports the given extension. It is not only checked if the parser can parse such files,
     * it is also checked if the extension is not included in the extension-deny list.
     * @param ext extension name
     * @return an error if the extension is not supported, null otherwise
     */
    public static String supportsExtension(final String ext) {
        if (ext == null || ext.isEmpty()) return null;
        if (denyExtensionx.containsKey(ext)) return "file extension '" + ext + "' is denied (2)";
        final String mimeType = ext2mime.get(ext);
        if (mimeType == null) return "no parser available";
        final Set<Parser> idiom = mime2parser.get(mimeType);
        assert idiom != null;
        if (idiom == null || idiom.isEmpty()) return "no parser available (internal error!)";
        return null;
    }

    /**
     * checks if the parser supports the given extension or the file at the specified url. It is not only checked if the parser can parse such files,
     * it is also checked if the extension is not included in the extension-deny list.
     * @param url url to check
     * @return an error if the extension is not supported, null otherwise
     */
    public static String supportsExtension(final MultiProtocolURL url) {
        return supportsExtension(MultiProtocolURL.getFileExtension(url.getFileName()));
    }

    public static String mimeOf(final MultiProtocolURL url) {
        return mimeOf(MultiProtocolURL.getFileExtension(url.getFileName()));
    }

    public static String mimeOf(final String ext) {
        return ext2mime.get(ext.toLowerCase(Locale.ROOT));
    }

    /**
     * Normalize a media type information string (can be a HTTP "Content-Type"
     * response header) : convert to lower case, remove any supplementary
     * parameters such as the encoding (charset name), and provide a default
     * value when null.
     *
     * @param mimeType
     *            raw information about media type, eventually provided by a
     *            HTTP "Content-Type" response header
     * @return a non null media type in lower case
     */
    public static String normalizeMimeType(String mimeType) {
        if (mimeType == null) {
            return "application/octet-stream";
        }
        mimeType = mimeType.toLowerCase(Locale.ROOT);
        final int pos = mimeType.indexOf(';');
        return ((pos < 0) ? mimeType.trim() : mimeType.substring(0, pos).trim());
    }

    public static void setDenyMime(final String denyList) {
        denyMime.clear();
        String n;
        for (final String s: CommonPattern.COMMA.split(denyList)) {
            n = normalizeMimeType(s);
            if (n != null && n.length() > 0) denyMime.put(n, v);
        }
    }

    public static String getDenyMime() {
        String s = "";
        for (final String d: denyMime.keySet()) s += d + ",";
        if (!s.isEmpty()) s = s.substring(0, s.length() - 1);
        return s;
    }

    public static void grantMime(final String mime, final boolean grant) {
        final String n = normalizeMimeType(mime);
        if (n == null || n.isEmpty()) return;
        if (grant) denyMime.remove(n); else denyMime.put(n, v);
    }

    public static void setDenyExtension(final String denyList) {
        denyExtensionx.clear();
        for (final String s: CommonPattern.COMMA.split(denyList)) denyExtensionx.put(s.trim(), v);
    }

    public static String getDenyExtension() {
        String s = "";
        for (final String d: denyExtensionx.keySet()) s += d + ",";
        if (!s.isEmpty()) s = s.substring(0, s.length() - 1);
        return s;
    }

    public static void grantExtension(final String ext, final boolean grant) {
        if (ext == null || ext.isEmpty()) return;
        if (grant) denyExtensionx.remove(ext); else denyExtensionx.put(ext, v);
    }

}
