diff --git a/dev/checkstyle/suppressions.xml b/dev/checkstyle/suppressions.xml index 765ddb4cb47..7b872af41f8 100644 --- a/dev/checkstyle/suppressions.xml +++ b/dev/checkstyle/suppressions.xml @@ -37,7 +37,7 @@ Portions Copyright (c) 2018-2019, Chris Fraire . + |Context\.java|HistoryContext\.java|Suggester\.java|AnalyzersInfo\.java" /> diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerFramework.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerFramework.java new file mode 100644 index 00000000000..41a0b0499b6 --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerFramework.java @@ -0,0 +1,402 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + */ +package org.opengrok.indexer.analysis; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.logging.Level; +import java.util.logging.Logger; +import org.opengrok.indexer.analysis.ada.AdaAnalyzerFactory; +import org.opengrok.indexer.analysis.archive.BZip2AnalyzerFactory; +import org.opengrok.indexer.analysis.archive.GZIPAnalyzerFactory; +import org.opengrok.indexer.analysis.archive.TarAnalyzerFactory; +import org.opengrok.indexer.analysis.archive.ZipAnalyzerFactory; +import org.opengrok.indexer.analysis.c.CAnalyzerFactory; +import org.opengrok.indexer.analysis.c.CxxAnalyzerFactory; +import org.opengrok.indexer.analysis.clojure.ClojureAnalyzerFactory; +import org.opengrok.indexer.analysis.csharp.CSharpAnalyzerFactory; +import org.opengrok.indexer.analysis.data.IgnorantAnalyzerFactory; +import org.opengrok.indexer.analysis.data.ImageAnalyzerFactory; +import org.opengrok.indexer.analysis.document.MandocAnalyzerFactory; +import org.opengrok.indexer.analysis.document.TroffAnalyzerFactory; +import org.opengrok.indexer.analysis.eiffel.EiffelAnalyzerFactory; +import org.opengrok.indexer.analysis.erlang.ErlangAnalyzerFactory; +import org.opengrok.indexer.analysis.executables.ELFAnalyzerFactory; +import org.opengrok.indexer.analysis.executables.JarAnalyzerFactory; +import org.opengrok.indexer.analysis.executables.JavaClassAnalyzerFactory; +import org.opengrok.indexer.analysis.fortran.FortranAnalyzerFactory; +import org.opengrok.indexer.analysis.golang.GolangAnalyzerFactory; +import org.opengrok.indexer.analysis.haskell.HaskellAnalyzerFactory; +import org.opengrok.indexer.analysis.java.JavaAnalyzerFactory; +import org.opengrok.indexer.analysis.javascript.JavaScriptAnalyzerFactory; +import org.opengrok.indexer.analysis.json.JsonAnalyzerFactory; +import org.opengrok.indexer.analysis.kotlin.KotlinAnalyzerFactory; +import org.opengrok.indexer.analysis.lisp.LispAnalyzerFactory; +import org.opengrok.indexer.analysis.lua.LuaAnalyzerFactory; +import org.opengrok.indexer.analysis.pascal.PascalAnalyzerFactory; +import org.opengrok.indexer.analysis.perl.PerlAnalyzerFactory; +import org.opengrok.indexer.analysis.php.PhpAnalyzerFactory; +import org.opengrok.indexer.analysis.plain.PlainAnalyzerFactory; +import org.opengrok.indexer.analysis.plain.XMLAnalyzerFactory; +import org.opengrok.indexer.analysis.powershell.PowershellAnalyzerFactory; +import org.opengrok.indexer.analysis.python.PythonAnalyzerFactory; +import org.opengrok.indexer.analysis.ruby.RubyAnalyzerFactory; +import org.opengrok.indexer.analysis.rust.RustAnalyzerFactory; +import org.opengrok.indexer.analysis.scala.ScalaAnalyzerFactory; +import org.opengrok.indexer.analysis.sh.ShAnalyzerFactory; +import org.opengrok.indexer.analysis.sql.PLSQLAnalyzerFactory; +import org.opengrok.indexer.analysis.sql.SQLAnalyzerFactory; +import org.opengrok.indexer.analysis.swift.SwiftAnalyzerFactory; +import org.opengrok.indexer.analysis.tcl.TclAnalyzerFactory; +import org.opengrok.indexer.analysis.uue.UuencodeAnalyzerFactory; +import org.opengrok.indexer.analysis.vb.VBAnalyzerFactory; +import org.opengrok.indexer.analysis.verilog.VerilogAnalyzerFactory; +import org.opengrok.indexer.framework.PluginFramework; +import org.opengrok.indexer.logger.LoggerFactory; + +/** + * A framework for loading the analyzers from .jar or .class files. + */ +public class AnalyzerFramework extends PluginFramework { + + private static final Logger LOGGER = LoggerFactory.getLogger(AnalyzerFramework.class); + + /** + * The default {@code FileAnalyzerFactory} instance. + */ + public static final AnalyzerFactory DEFAULT_ANALYZER_FACTORY = new FileAnalyzerFactory(); + + private static final IAnalyzerPlugin[] DEFAULT_PLUGINS = { + () -> DEFAULT_ANALYZER_FACTORY, + IgnorantAnalyzerFactory::new, + BZip2AnalyzerFactory::new, + XMLAnalyzerFactory::new, + () -> MandocAnalyzerFactory.DEFAULT_INSTANCE, + () -> TroffAnalyzerFactory.DEFAULT_INSTANCE, + ELFAnalyzerFactory::new, + () -> JavaClassAnalyzerFactory.DEFAULT_INSTANCE, + ImageAnalyzerFactory::new, + () -> JarAnalyzerFactory.DEFAULT_INSTANCE, + () -> ZipAnalyzerFactory.DEFAULT_INSTANCE, + TarAnalyzerFactory::new, + CAnalyzerFactory::new, + CSharpAnalyzerFactory::new, + VBAnalyzerFactory::new, + CxxAnalyzerFactory::new, + ErlangAnalyzerFactory::new, + ShAnalyzerFactory::new, + PowershellAnalyzerFactory::new, + () -> PlainAnalyzerFactory.DEFAULT_INSTANCE, + UuencodeAnalyzerFactory::new, + GZIPAnalyzerFactory::new, + JavaAnalyzerFactory::new, + JavaScriptAnalyzerFactory::new, + KotlinAnalyzerFactory::new, + SwiftAnalyzerFactory::new, + JsonAnalyzerFactory::new, + PythonAnalyzerFactory::new, + RustAnalyzerFactory::new, + PerlAnalyzerFactory::new, + PhpAnalyzerFactory::new, + LispAnalyzerFactory::new, + TclAnalyzerFactory::new, + ScalaAnalyzerFactory::new, + ClojureAnalyzerFactory::new, + SQLAnalyzerFactory::new, + PLSQLAnalyzerFactory::new, + FortranAnalyzerFactory::new, + HaskellAnalyzerFactory::new, + GolangAnalyzerFactory::new, + LuaAnalyzerFactory::new, + PascalAnalyzerFactory::new, + AdaAnalyzerFactory::new, + RubyAnalyzerFactory::new, + EiffelAnalyzerFactory::new, + VerilogAnalyzerFactory::new + }; + + /** + * Instance of analyzers info, which should be used in the outside world. + */ + private AtomicReference analyzersInfo = new AtomicReference<>(new AnalyzersInfo()); + + /** + * List of lambdas to perform customizations to the analyzers info when the {@link #reload()} + * is in progress. This is needed because customizations would be lost with next + * call to {@link #reload()}. + * + * @see #reload() + */ + private List> customizations = new LinkedList<>(); + + + /** + * Construct the analyzer framework with a plugin directory. + *

+ * NOTE: This call ensures the framework is calling {@link #reload()} with default plugins. + * + * @param pluginDirectory directory with plugins + * @see #reload() + */ + public AnalyzerFramework(String pluginDirectory) { + super(IAnalyzerPlugin.class, pluginDirectory); + reload(); + } + + /** + * Get the current analyzers info. + * + * @return current instance of analyzers info, never {@code null}. + */ + public AnalyzersInfo getAnalyzersInfo() { + return analyzersInfo.get(); + } + + /** + * Prepare the analyzer plugin and register the analyzer to the analyzers info. + * + * @param localInfo an analyzers info used for loading the plugins + * @param plugin the loaded plugin interface with the factory. + */ + @Override + protected void classLoaded(AnalyzersInfo localInfo, IAnalyzerPlugin plugin) { + final AnalyzerFactory analyzer = plugin.getFactory(); + + registerAnalyzer(localInfo, analyzer); + + if (analyzer.getName() != null && !analyzer.getName().isEmpty()) { + localInfo.fileTypeDescriptions.put(analyzer.getAnalyzer().getFileTypeName(), analyzer.getName()); + } + + LOGGER.log(Level.FINER, "An analyzer factory {0} has been loaded.", analyzer.getClass().getCanonicalName()); + } + + /** + * Prepare the loading info and load the default plugins. + * + * @return an analyzers info used for loading the plugins + * @see #DEFAULT_PLUGINS + */ + @Override + protected AnalyzersInfo beforeReload() { + final AnalyzersInfo localInfo = new AnalyzersInfo(); + for (IAnalyzerPlugin plugin : DEFAULT_PLUGINS) { + classLoaded(localInfo, plugin); + } + return localInfo; + } + + /** + * Run the customizations and swap the instance in {@code analyzersInfo}. + * + * @param localInfo an analyzers info used for loading the plugins + * @see #customizations + * @see #analyzersInfo + */ + @Override + protected void afterReload(AnalyzersInfo localInfo) { + if (getPluginDirectory() == null || !getPluginDirectory().isDirectory() || !getPluginDirectory().canRead()) { + LOGGER.log(Level.WARNING, "No plugin directory for analyzers."); + } + + // apply custom settings for this framework + customizations.stream().forEach(customization -> customization.accept(localInfo)); + + this.analyzersInfo.set(localInfo.freeze()); + } + + /** + * Register the analyzer factory in the framework. + *

+ * NOTE: This method calls {@link #reload()} so it may take some time to complete. + * + * @param factory the analyzer factory + */ + public void registerAnalyzer(AnalyzerFactory factory) { + customizations.add(analyzersInfo -> { + registerAnalyzer(analyzersInfo, factory); + }); + + reload(); + } + + /** + * Register a {@code FileAnalyzerFactory} instance into the given {@code analyzersInfo}. + */ + private static void registerAnalyzer(AnalyzersInfo analyzersInfo, AnalyzerFactory factory) { + for (String name : factory.getFileNames()) { + AnalyzerFactory old = analyzersInfo.fileNames.put(name, factory); + assert old == null : + "name '" + name + "' used in multiple analyzers"; + } + for (String prefix : factory.getPrefixes()) { + AnalyzerFactory old = analyzersInfo.prefixes.put(prefix, factory); + assert old == null : + "prefix '" + prefix + "' used in multiple analyzers"; + } + for (String suffix : factory.getSuffixes()) { + AnalyzerFactory old = analyzersInfo.extensions.put(suffix, factory); + assert old == null : + "suffix '" + suffix + "' used in multiple analyzers"; + } + for (String magic : factory.getMagicStrings()) { + AnalyzerFactory old = analyzersInfo.magics.put(magic, factory); + assert old == null : + "magic '" + magic + "' used in multiple analyzers"; + } + + analyzersInfo.matchers.addAll(factory.getMatchers()); + analyzersInfo.factories.add(factory); + + AbstractAnalyzer fa = factory.getAnalyzer(); + String fileTypeName = fa.getFileTypeName(); + analyzersInfo.filetypeFactories.put(fileTypeName, factory); + analyzersInfo.analyzerVersions.put(fileTypeName, fa.getVersionNo()); + } + + + /** + * Instruct the AnalyzerGuru to use a given analyzer for a given file prefix. + *

+ * NOTE: This method calls {@link #reload()} so it may take some time to complete. + * + * @param prefix the file prefix to add + * @param factory a factory which creates the analyzer to use for the given + * extension (if you pass null as the analyzer, you will disable the + * analyzer used for that extension) + */ + public void addPrefix(String prefix, AnalyzerFactory factory) { + customizations.add(analyzersInfo -> { + AnalyzerFactory oldFactory; + if (factory == null) { + oldFactory = analyzersInfo.prefixes.remove(prefix); + LOGGER.log(Level.FINER, + "Removing a mapping for prefix {0}{1}", + new Object[]{ + prefix, + oldFactory != null ? String.format(". And has removed existing factory '%s'.", oldFactory.getClass().getCanonicalName()) : "" + } + ); + + } else { + oldFactory = analyzersInfo.prefixes.put(prefix, factory); + LOGGER.log(Level.FINER, + "Adding a factory {0} for matching prefix {1}{2}", + new Object[]{ + factory.getClass().getCanonicalName(), + prefix, + oldFactory != null ? String.format(". And has replaced existing factory '%s'.", oldFactory.getClass().getCanonicalName()) : "" + + } + ); + } + + if (factoriesDifferent(factory, oldFactory)) { + analyzersInfo.customizations.add("p:" + prefix); + } + }); + + reload(); + } + + /** + * Instruct the AnalyzerGuru to use a given analyzer for a given file extension. + *

+ * NOTE: This method calls {@link #reload()} so it may take some time to complete. + * + * @param extension the file-extension to add + * @param factory a factory which creates the analyzer to use for the given + * extension (if you pass null as the analyzer, you will disable the + * analyzer used for that extension) + */ + public void addExtension(String extension, AnalyzerFactory factory) { + customizations.add(analyzersInfo -> { + AnalyzerFactory oldFactory; + if (factory == null) { + oldFactory = analyzersInfo.extensions.remove(extension); + LOGGER.log(Level.FINER, + "Removing a mapping for suffix {0}{1}", + new Object[]{ + extension, + oldFactory != null ? String.format(". And has removed existing factory '%s'.", oldFactory.getClass().getCanonicalName()) : "" + } + ); + } else { + oldFactory = analyzersInfo.extensions.put(extension, factory); + LOGGER.log(Level.FINER, + "Adding a factory {0} for matching suffix {1}{2}", + new Object[]{ + factory.getClass().getCanonicalName(), + extension, + oldFactory != null ? String.format(". And has replaced existing factory '%s'.", oldFactory.getClass().getCanonicalName()) : "" + } + ); + } + + if (factoriesDifferent(factory, oldFactory)) { + analyzersInfo.customizations.add("s:" + extension); + } + }); + + reload(); + } + + + /** + * Free resources associated with all registered analyzers. + */ + public void returnAnalyzers() { + getAnalyzersInfo().factories.forEach(AnalyzerFactory::returnAnalyzer); + } + + /** + * Decide if two factory instances are different. + * + * @param a the first instance + * @param b the second instance + * @return true if we consider them different, false otherwise + */ + private static boolean factoriesDifferent(AnalyzerFactory a, AnalyzerFactory b) { + String a_name = null; + if (a != null) { + a_name = a.getName(); + if (a_name == null) { + a_name = a.getClass().getSimpleName(); + } + } + String b_name = null; + if (b != null) { + b_name = b.getName(); + if (b_name == null) { + b_name = b.getClass().getSimpleName(); + } + } + if (a_name == null && b_name == null) { + return false; + } + return a_name == null || b_name == null || !a_name.equals(b_name); + } + +} diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java index f1863a03de6..0c41732b9de 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuru.java @@ -30,22 +30,15 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; -import java.io.StringReader; import java.io.Writer; import java.lang.reflect.InvocationTargetException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.SortedMap; -import java.util.TreeMap; -import java.util.TreeSet; import java.util.logging.Level; import java.util.logging.Logger; import org.apache.lucene.document.DateTools; @@ -57,52 +50,6 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.util.BytesRef; -import org.opengrok.indexer.analysis.FileAnalyzerFactory.Matcher; -import org.opengrok.indexer.analysis.ada.AdaAnalyzerFactory; -import org.opengrok.indexer.analysis.archive.BZip2AnalyzerFactory; -import org.opengrok.indexer.analysis.archive.GZIPAnalyzerFactory; -import org.opengrok.indexer.analysis.archive.TarAnalyzerFactory; -import org.opengrok.indexer.analysis.archive.ZipAnalyzerFactory; -import org.opengrok.indexer.analysis.c.CAnalyzerFactory; -import org.opengrok.indexer.analysis.c.CxxAnalyzerFactory; -import org.opengrok.indexer.analysis.clojure.ClojureAnalyzerFactory; -import org.opengrok.indexer.analysis.csharp.CSharpAnalyzerFactory; -import org.opengrok.indexer.analysis.data.IgnorantAnalyzerFactory; -import org.opengrok.indexer.analysis.data.ImageAnalyzerFactory; -import org.opengrok.indexer.analysis.document.MandocAnalyzerFactory; -import org.opengrok.indexer.analysis.document.TroffAnalyzerFactory; -import org.opengrok.indexer.analysis.eiffel.EiffelAnalyzerFactory; -import org.opengrok.indexer.analysis.erlang.ErlangAnalyzerFactory; -import org.opengrok.indexer.analysis.executables.ELFAnalyzerFactory; -import org.opengrok.indexer.analysis.executables.JarAnalyzerFactory; -import org.opengrok.indexer.analysis.executables.JavaClassAnalyzerFactory; -import org.opengrok.indexer.analysis.fortran.FortranAnalyzerFactory; -import org.opengrok.indexer.analysis.golang.GolangAnalyzerFactory; -import org.opengrok.indexer.analysis.haskell.HaskellAnalyzerFactory; -import org.opengrok.indexer.analysis.java.JavaAnalyzerFactory; -import org.opengrok.indexer.analysis.javascript.JavaScriptAnalyzerFactory; -import org.opengrok.indexer.analysis.json.JsonAnalyzerFactory; -import org.opengrok.indexer.analysis.kotlin.KotlinAnalyzerFactory; -import org.opengrok.indexer.analysis.lisp.LispAnalyzerFactory; -import org.opengrok.indexer.analysis.lua.LuaAnalyzerFactory; -import org.opengrok.indexer.analysis.pascal.PascalAnalyzerFactory; -import org.opengrok.indexer.analysis.perl.PerlAnalyzerFactory; -import org.opengrok.indexer.analysis.php.PhpAnalyzerFactory; -import org.opengrok.indexer.analysis.plain.PlainAnalyzerFactory; -import org.opengrok.indexer.analysis.plain.XMLAnalyzerFactory; -import org.opengrok.indexer.analysis.powershell.PowershellAnalyzerFactory; -import org.opengrok.indexer.analysis.python.PythonAnalyzerFactory; -import org.opengrok.indexer.analysis.ruby.RubyAnalyzerFactory; -import org.opengrok.indexer.analysis.rust.RustAnalyzerFactory; -import org.opengrok.indexer.analysis.scala.ScalaAnalyzerFactory; -import org.opengrok.indexer.analysis.sh.ShAnalyzerFactory; -import org.opengrok.indexer.analysis.sql.PLSQLAnalyzerFactory; -import org.opengrok.indexer.analysis.sql.SQLAnalyzerFactory; -import org.opengrok.indexer.analysis.swift.SwiftAnalyzerFactory; -import org.opengrok.indexer.analysis.tcl.TclAnalyzerFactory; -import org.opengrok.indexer.analysis.uue.UuencodeAnalyzerFactory; -import org.opengrok.indexer.analysis.vb.VBAnalyzerFactory; -import org.opengrok.indexer.analysis.verilog.VerilogAnalyzerFactory; import org.opengrok.indexer.configuration.Project; import org.opengrok.indexer.configuration.RuntimeEnvironment; import org.opengrok.indexer.history.Annotation; @@ -119,26 +66,26 @@ * Manages and provides Analyzers as needed. Please see * * this page for a great description of the purpose of the AnalyzerGuru. - * + *

* Created on September 22, 2005 * * @author Chandan */ public class AnalyzerGuru { + private static final Logger LOGGER = LoggerFactory.getLogger(AnalyzerGuru.class); + /** * The maximum number of characters (multi-byte if a BOM is identified) to * read from the input stream to be used for magic string matching */ private static final int OPENING_MAX_CHARS = 100; - /** * Set to 16K -- though debugging shows it would do with only 8K+3 * (standard buffer for Java BufferedInputStream plus 3 bytes for largest - * UTF BOM) + * UTF BOM) */ private static final int MARK_READ_LIMIT = 1024 * 16; - /** * The number of bytes read from the start of the file for magic number or * string analysis. Some {@link FileAnalyzerFactory.Matcher} @@ -147,100 +94,14 @@ public class AnalyzerGuru { */ private static final int MAGIC_BYTES_NUM = 8; - private static final Logger LOGGER = LoggerFactory.getLogger(AnalyzerGuru.class); - - /** - * The default {@code FileAnalyzerFactory} instance. - */ - private static final AnalyzerFactory DEFAULT_ANALYZER_FACTORY = new FileAnalyzerFactory(); - - /** - * Map from file names to analyzer factories. - */ - private static final Map FILE_NAMES = new HashMap<>(); - - /** - * Map from file extensions to analyzer factories. - */ - private static final Map ext = new HashMap<>(); - - /** - * Map from file prefixes to analyzer factories. - */ - private static final Map pre = new HashMap<>(); - - /** - * Appended when - * {@link #addExtension(java.lang.String, AnalyzerFactory)} - * or - * {@link #addPrefix(java.lang.String, AnalyzerFactory)} - * are called to augment the value in {@link #getVersionNo()}. - */ - private static final TreeSet CUSTOMIZATION_KEYS = new TreeSet<>(); - - private static int customizationHashCode; - - /** - * Descending string length comparator for magics - */ - private static final Comparator descStrlenComparator = - new Comparator() { - @Override public int compare(String s1, String s2) { - // DESC: s2 length <=> s1 length - int cmp = Integer.compare(s2.length(), s1.length()); - if (cmp != 0) { - return cmp; - } - - // the Comparator must also be "consistent with equals", so check - // string contents too when (length)cmp == 0. (ASC: s1 <=> s2.) - cmp = s1.compareTo(s2); - return cmp; - } - }; - - /** - * Map from magic strings to analyzer factories. - */ - private static final SortedMap magics = - new TreeMap<>(descStrlenComparator); - - /** - * List of matcher objects which can be used to determine which analyzer - * factory to use. - */ - private static final List matchers = new ArrayList<>(); - - /** - * List of all registered {@code FileAnalyzerFactory} instances. - */ - private static final List factories = new ArrayList<>(); - /** * Names of all analysis packages. */ private static final List analysisPkgNames = new ArrayList<>(); - public static final Reader dummyR = new StringReader(""); - public static final String dummyS = ""; public static final FieldType string_ft_stored_nanalyzed_norms = new FieldType(StringField.TYPE_STORED); public static final FieldType string_ft_nstored_nanalyzed_norms = new FieldType(StringField.TYPE_NOT_STORED); - private static final Map fileTypeDescriptions = new TreeMap<>(); - - /** - * Maps from {@link FileAnalyzer#getFileTypeName()} to - * {@link FileAnalyzerFactory} - */ - private static final Map FILETYPE_FACTORIES = - new HashMap<>(); - - /** - * Maps from {@link FileAnalyzer#getFileTypeName()} to - * {@link FileAnalyzer#getVersionNo()} - */ - private static final Map ANALYZER_VERSIONS = new HashMap<>(); - /* * If you write your own analyzer please register it here. The order is * important for any factory that uses a FileAnalyzerFactory.Matcher @@ -249,74 +110,33 @@ public class AnalyzerGuru { */ static { try { - AnalyzerFactory[] analyzers = { - DEFAULT_ANALYZER_FACTORY, - new IgnorantAnalyzerFactory(), - new BZip2AnalyzerFactory(), - new XMLAnalyzerFactory(), - MandocAnalyzerFactory.DEFAULT_INSTANCE, - TroffAnalyzerFactory.DEFAULT_INSTANCE, - new ELFAnalyzerFactory(), - JavaClassAnalyzerFactory.DEFAULT_INSTANCE, - new ImageAnalyzerFactory(), - JarAnalyzerFactory.DEFAULT_INSTANCE, - ZipAnalyzerFactory.DEFAULT_INSTANCE, - new TarAnalyzerFactory(), - new CAnalyzerFactory(), - new CSharpAnalyzerFactory(), - new VBAnalyzerFactory(), - new CxxAnalyzerFactory(), - new ErlangAnalyzerFactory(), - new ShAnalyzerFactory(), - new PowershellAnalyzerFactory(), - PlainAnalyzerFactory.DEFAULT_INSTANCE, - new UuencodeAnalyzerFactory(), - new GZIPAnalyzerFactory(), - new JavaAnalyzerFactory(), - new JavaScriptAnalyzerFactory(), - new KotlinAnalyzerFactory(), - new SwiftAnalyzerFactory(), - new JsonAnalyzerFactory(), - new PythonAnalyzerFactory(), - new RustAnalyzerFactory(), - new PerlAnalyzerFactory(), - new PhpAnalyzerFactory(), - new LispAnalyzerFactory(), - new TclAnalyzerFactory(), - new ScalaAnalyzerFactory(), - new ClojureAnalyzerFactory(), - new SQLAnalyzerFactory(), - new PLSQLAnalyzerFactory(), - new FortranAnalyzerFactory(), - new HaskellAnalyzerFactory(), - new GolangAnalyzerFactory(), - new LuaAnalyzerFactory(), - new PascalAnalyzerFactory(), - new AdaAnalyzerFactory(), - new RubyAnalyzerFactory(), - new EiffelAnalyzerFactory(), - new VerilogAnalyzerFactory() - }; - - for (AnalyzerFactory analyzer : analyzers) { - registerAnalyzer(analyzer); - } - - for (AnalyzerFactory analyzer : analyzers) { - if (analyzer.getName() != null && !analyzer.getName().isEmpty()) { - fileTypeDescriptions.put(analyzer.getAnalyzer().getFileTypeName(), analyzer.getName()); - } - } - string_ft_stored_nanalyzed_norms.setOmitNorms(false); string_ft_nstored_nanalyzed_norms.setOmitNorms(false); - } catch (Throwable t) { + } catch (IllegalStateException t) { LOGGER.log(Level.SEVERE, "exception hit when constructing AnalyzerGuru static", t); throw t; } } + private final AnalyzerFramework pluginFramework; + + /** + * Create a new instance of analyzer guru with default analyzers. + */ + public AnalyzerGuru() { + pluginFramework = new AnalyzerFramework(null); + } + + /** + * Create a new instance of analyzer guru with default instance and loading other analyzers from a plugin directory. + * + * @param pluginDirectory path to the plugin directory + */ + public AnalyzerGuru(String pluginDirectory) { + pluginFramework = new AnalyzerFramework(pluginDirectory); + } + /** * Gets a version number to be used to tag documents examined by the guru so * that {@link AbstractAnalyzer} selection can be re-done later if a stored @@ -328,19 +148,21 @@ public class AnalyzerGuru { * {@link FileAnalyzerFactory} subclasses are registered or when existing * {@link FileAnalyzerFactory} subclasses are revised to target more or * different files. + * * @return a value whose lower 32-bits are a static value * 20190211_00 * for the current implementation and whose higher-32 bits are non-zero if - * {@link #addExtension(java.lang.String, AnalyzerFactory)} + * {@link #addExtension(String, AnalyzerFactory)} * or - * {@link #addPrefix(java.lang.String, AnalyzerFactory)} + * {@link #addPrefix(String, AnalyzerFactory)} * has been called. */ - public static long getVersionNo() { + public long getVersionNo() { final int ver32 = 20190211_00; // Edit comment above too! long ver = ver32; - if (customizationHashCode != 0) { - ver |= (long)customizationHashCode << 32; + final int customizations = Objects.hash(this.pluginFramework.getAnalyzersInfo().customizations.toArray()); + if (customizations != 0) { + ver |= (long) customizations << 32; } return ver; } @@ -349,348 +171,57 @@ public static long getVersionNo() { * Gets a version number according to a registered * {@link FileAnalyzer#getVersionNo()} for a {@code fileTypeName} according * to {@link FileAnalyzer#getFileTypeName()}. + * * @param fileTypeName a defined instance * @return a registered value or {@link Long#MIN_VALUE} if * {@code fileTypeName} is unknown */ - public static long getAnalyzerVersionNo(String fileTypeName) { - return ANALYZER_VERSIONS.getOrDefault(fileTypeName, Long.MIN_VALUE); - } - - public static Map getAnalyzersVersionNos() { - return Collections.unmodifiableMap(ANALYZER_VERSIONS); - } - - public static Map getExtensionsMap() { - return Collections.unmodifiableMap(ext); - } - - public static Map getPrefixesMap() { - return Collections.unmodifiableMap(pre); + public long getAnalyzerVersionNo(String fileTypeName) { + return this.pluginFramework.getAnalyzersInfo().analyzerVersions.getOrDefault(fileTypeName, Long.MIN_VALUE); } - public static Map getMagicsMap() { - return Collections.unmodifiableMap(magics); + public Map getAnalyzersVersionNos() { + return this.pluginFramework.getAnalyzersInfo().analyzerVersions; } - public static List getAnalyzerFactoryMatchers() { - return Collections.unmodifiableList(matchers); + public Map getExtensionsMap() { + return this.pluginFramework.getAnalyzersInfo().extensions; } - public static Map getfileTypeDescriptions() { - return Collections.unmodifiableMap(fileTypeDescriptions); - } - - public List getAnalyzerFactories() { - return Collections.unmodifiableList(factories); - } - - /** - * Register a {@code FileAnalyzerFactory} instance. - */ - private static void registerAnalyzer(AnalyzerFactory factory) { - for (String name : factory.getFileNames()) { - AnalyzerFactory old = FILE_NAMES.put(name, factory); - assert old == null : - "name '" + name + "' used in multiple analyzers"; - } - for (String prefix : factory.getPrefixes()) { - AnalyzerFactory old = pre.put(prefix, factory); - assert old == null : - "prefix '" + prefix + "' used in multiple analyzers"; - } - for (String suffix : factory.getSuffixes()) { - AnalyzerFactory old = ext.put(suffix, factory); - assert old == null : - "suffix '" + suffix + "' used in multiple analyzers"; - } - for (String magic : factory.getMagicStrings()) { - AnalyzerFactory old = magics.put(magic, factory); - assert old == null : - "magic '" + magic + "' used in multiple analyzers"; - } - matchers.addAll(factory.getMatchers()); - factories.add(factory); - - AbstractAnalyzer fa = factory.getAnalyzer(); - String fileTypeName = fa.getFileTypeName(); - FILETYPE_FACTORIES.put(fileTypeName, factory); - ANALYZER_VERSIONS.put(fileTypeName, fa.getVersionNo()); + public Map getPrefixesMap() { + return this.pluginFramework.getAnalyzersInfo().prefixes; } - /** - * Instruct the AnalyzerGuru to use a given analyzer for a given file - * prefix. - * - * @param prefix the file prefix to add - * @param factory a factory which creates the analyzer to use for the given - * extension (if you pass null as the analyzer, you will disable the - * analyzer used for that extension) - */ - public static void addPrefix(String prefix, AnalyzerFactory factory) { - AnalyzerFactory oldFactory; - if (factory == null) { - oldFactory = pre.remove(prefix); - } else { - oldFactory = pre.put(prefix, factory); - } - - if (factoriesDifferent(factory, oldFactory)) { - addCustomizationKey("p:" + prefix); - } + public Map getMagicsMap() { + return this.pluginFramework.getAnalyzersInfo().magics; } - /** - * Instruct the AnalyzerGuru to use a given analyzer for a given file - * extension. - * - * @param extension the file-extension to add - * @param factory a factory which creates the analyzer to use for the given - * extension (if you pass null as the analyzer, you will disable the - * analyzer used for that extension) - */ - public static void addExtension(String extension, - AnalyzerFactory factory) { - AnalyzerFactory oldFactory; - if (factory == null) { - oldFactory = ext.remove(extension); - } else { - oldFactory = ext.put(extension, factory); - } - - if (factoriesDifferent(factory, oldFactory)) { - addCustomizationKey("e:" + extension); - } - } - - /** - * Get the default Analyzer. - * - * @return default FileAnalyzer - */ - public static AbstractAnalyzer getAnalyzer() { - return DEFAULT_ANALYZER_FACTORY.getAnalyzer(); - } - - /** - * Gets an analyzer for the specified {@code fileTypeName} if it accords - * with a known {@link FileAnalyzer#getFileTypeName()}. - * @param fileTypeName a defined name - * @return a defined instance if known or otherwise {@code null} - */ - public static AbstractAnalyzer getAnalyzer(String fileTypeName) { - AnalyzerFactory factory = FILETYPE_FACTORIES.get(fileTypeName); - return factory == null ? null : factory.getAnalyzer(); - } - - /** - * Get an analyzer suited to analyze a file. This function will reuse - * analyzers since they are costly. - * - * @param in Input stream containing data to be analyzed - * @param file Name of the file to be analyzed - * @return An analyzer suited for that file content - * @throws java.io.IOException If an error occurs while accessing the data - * in the input stream. - */ - public static AbstractAnalyzer getAnalyzer(InputStream in, String file) throws IOException { - AnalyzerFactory factory = find(in, file); - if (factory == null) { - AbstractAnalyzer defaultAnalyzer = getAnalyzer(); - if (LOGGER.isLoggable(Level.FINE)) { - LOGGER.log(Level.FINE, "{0}: fallback {1}", - new Object[]{file, - defaultAnalyzer.getClass().getSimpleName() }); - } - return defaultAnalyzer; - } - return factory.getAnalyzer(); - } - - /** - * Free resources associated with all registered analyzers. - */ - public static void returnAnalyzers() { - for (AnalyzerFactory analyzer : factories) { - analyzer.returnAnalyzer(); - } - } - - /** - * Populate a Lucene document with the required fields. - * - * @param doc The document to populate - * @param file The file to index - * @param path Where the file is located (from source root) - * @param fa The analyzer to use on the file - * @param xrefOut Where to write the xref (possibly {@code null}) - * @throws IOException If an exception occurs while collecting the data - * @throws InterruptedException if a timeout occurs - * @throws ForbiddenSymlinkException if symbolic-link checking encounters - * an ineligible link - */ - public void populateDocument(Document doc, File file, String path, - AbstractAnalyzer fa, Writer xrefOut) throws IOException, - InterruptedException, ForbiddenSymlinkException { - - String date = DateTools.timeToString(file.lastModified(), - DateTools.Resolution.MILLISECOND); - path = Util.fixPathIfWindows(path); - doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), - string_ft_stored_nanalyzed_norms)); - doc.add(new Field(QueryBuilder.FULLPATH, file.getAbsolutePath(), - string_ft_nstored_nanalyzed_norms)); - doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, - new BytesRef(file.getAbsolutePath()))); - - if (RuntimeEnvironment.getInstance().isHistoryEnabled()) { - try { - HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file); - if (hr != null) { - doc.add(new TextField(QueryBuilder.HIST, hr)); - // date = hr.getLastCommentDate() //RFE - } - } catch (HistoryException e) { - LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e); - } - } - doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms)); - doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date))); - - // `path' is not null, as it was passed to Util.path2uid() above. - doc.add(new TextField(QueryBuilder.PATH, path, Store.YES)); - Project project = Project.getProject(path); - if (project != null) { - doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES)); - } - - /* - * Use the parent of the path -- not the absolute file as is done for - * FULLPATH -- so that DIRPATH is the same convention as for PATH - * above. A StringField, however, is used instead of a TextField. - */ - File fpath = new File(path); - String fileParent = fpath.getParent(); - if (fileParent != null && fileParent.length() > 0) { - String normalizedPath = QueryBuilder.normalizeDirPath(fileParent); - StringField npstring = new StringField(QueryBuilder.DIRPATH, - normalizedPath, Store.NO); - doc.add(npstring); - } - - if (fa != null) { - AbstractAnalyzer.Genre g = fa.getGenre(); - if (g == AbstractAnalyzer.Genre.PLAIN || g == AbstractAnalyzer.Genre.XREFABLE || g == AbstractAnalyzer.Genre.HTML) { - doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms)); - } - fa.analyze(doc, StreamSource.fromFile(file), xrefOut); - - String type = fa.getFileTypeName(); - doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES)); - } - } - - /** - * Get the content type for a named file. - * - * @param in The input stream we want to get the content type for (if we - * cannot determine the content type by the filename) - * @param file The name of the file - * @return The contentType suitable for printing to - * response.setContentType() or null if the factory was not found - * @throws java.io.IOException If an error occurs while accessing the input - * stream. - */ - public static String getContentType(InputStream in, String file) throws IOException { - AnalyzerFactory factory = find(in, file); - String type = null; - if (factory != null) { - type = factory.getContentType(); - } - return type; - } - - /** - * Write a browse-able version of the file - * - * @param factory The analyzer factory for this file type - * @param in The input stream containing the data - * @param out Where to write the result - * @param defs definitions for the source file, if available - * @param annotation Annotation information for the file - * @param project Project the file belongs to - * @throws java.io.IOException If an error occurs while creating the output - */ - public static void writeXref(AnalyzerFactory factory, Reader in, - Writer out, Definitions defs, - Annotation annotation, Project project) - throws IOException { - Reader input = in; - if (factory.getGenre() == AbstractAnalyzer.Genre.PLAIN) { - // This is some kind of text file, so we need to expand tabs to - // spaces to match the project's tab settings. - input = ExpandTabsReader.wrap(in, project); - } - - WriteXrefArgs args = new WriteXrefArgs(input, out); - args.setDefs(defs); - args.setAnnotation(annotation); - args.setProject(project); - - AbstractAnalyzer analyzer = factory.getAnalyzer(); - RuntimeEnvironment env = RuntimeEnvironment.getInstance(); - analyzer.setScopesEnabled(env.isScopesEnabled()); - analyzer.setFoldingEnabled(env.isFoldingEnabled()); - analyzer.writeXref(args); + public List getAnalyzerFactoryMatchers() { + return this.pluginFramework.getAnalyzersInfo().matchers; } - /** - * Writes a browse-able version of the file transformed for immediate - * serving to a web client. - * @param contextPath the web context path for - * {@link Util#dumpXref(java.io.Writer, java.io.Reader, java.lang.String)} - * @param factory the analyzer factory for this file type - * @param in the input stream containing the data - * @param out a defined instance to write - * @param defs definitions for the source file, if available - * @param annotation annotation information for the file - * @param project project the file belongs to - * @throws java.io.IOException if an error occurs while creating the output - */ - public static void writeDumpedXref(String contextPath, - AnalyzerFactory factory, Reader in, Writer out, - Definitions defs, Annotation annotation, Project project) - throws IOException { - - File xrefTemp = File.createTempFile("ogxref", ".html"); - try { - try (FileWriter tmpout = new FileWriter(xrefTemp)) { - writeXref(factory, in, tmpout, defs, annotation, project); - } - Util.dumpXref(out, xrefTemp, false, contextPath); - } finally { - xrefTemp.delete(); - } + public Map getfileTypeDescriptions() { + return this.pluginFramework.getAnalyzersInfo().fileTypeDescriptions; } /** - * Get the genre of a file + * Get the genre of a file. * * @param file The file to inspect * @return The genre suitable to decide how to display the file */ - public static AbstractAnalyzer.Genre getGenre(String file) { + public AbstractAnalyzer.Genre getGenre(String file) { return getGenre(find(file)); } /** - * Get the genre of a bulk of data + * Get the genre of a bulk of data. * * @param in A stream containing the data * @return The genre suitable to decide how to display the file * @throws java.io.IOException If an error occurs while getting the content */ - public static AbstractAnalyzer.Genre getGenre(InputStream in) throws IOException { + public AbstractAnalyzer.Genre getGenre(InputStream in) throws IOException { return getGenre(find(in)); } @@ -710,11 +241,12 @@ public static AbstractAnalyzer.Genre getGenre(AnalyzerFactory factory) { /** * Finds a {@code FileAnalyzerFactory} for the specified * {@link FileAnalyzer#getFileTypeName()}. + * * @param fileTypeName a defined instance * @return a defined instance or {@code null} */ - public static AnalyzerFactory findByFileTypeName(String fileTypeName) { - return FILETYPE_FACTORIES.get(fileTypeName); + public AnalyzerFactory findByFileTypeName(String fileTypeName) { + return this.pluginFramework.getAnalyzersInfo().filetypeFactories.get(fileTypeName); } /** @@ -725,66 +257,63 @@ public static AnalyzerFactory findByFileTypeName(String fileTypeName) { * * @param factoryClassName name of the factory class * @return a file analyzer factory - * - * @throws ClassNotFoundException if there is no class with that name - * @throws ClassCastException if the class is not a subclass of {@code - * FileAnalyzerFactory} - * @throws IllegalAccessException if the constructor cannot be accessed - * @throws InstantiationException if the class cannot be instantiated - * @throws NoSuchMethodException if no-argument constructor could not be found + * @throws ClassNotFoundException if there is no class with that name + * @throws ClassCastException if the class is not a subclass of {@code + * FileAnalyzerFactory} + * @throws IllegalAccessException if the constructor cannot be accessed + * @throws InstantiationException if the class cannot be instantiated + * @throws NoSuchMethodException if no-argument constructor could not be found * @throws InvocationTargetException if the underlying constructor throws an exception */ - public static AnalyzerFactory findFactory(String factoryClassName) - throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, + public AnalyzerFactory findFactory(String factoryClassName) + throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException { Class fcn = null; try { fcn = Class.forName(factoryClassName); - } catch (ClassNotFoundException e) { fcn = getFactoryClass(factoryClassName); - + if (fcn == null) { throw new ClassNotFoundException("Unable to locate class " + factoryClassName); } } - + return findFactory(fcn); } /** * Get Analyzer factory class using class simple name. - * + * * @param simpleName which may be either the factory class - * simple name (eg. CAnalyzerFactory), the analyzer name - * (eg. CAnalyzer), or the language name (eg. C) - * + * simple name (eg. CAnalyzerFactory), the analyzer name + * (eg. CAnalyzer), or the language name (eg. C) * @return the analyzer factory class, or null when not found. */ - public static Class getFactoryClass(String simpleName) { + public Class getFactoryClass(String simpleName) { Class factoryClass = null; - + // Build analysis package name list first time only if (analysisPkgNames.isEmpty()) { Package[] p = Package.getPackages(); - for(Package pp : p){ + for (Package pp : p) { String pname = pp.getName(); if (pname.indexOf(".analysis.") != -1) { analysisPkgNames.add(pname); } } } - + // This allows user to enter the language or analyzer name // (eg. C or CAnalyzer vs. CAnalyzerFactory) // Note that this assumes a regular naming scheme of - // all language parsers: + // all language parsers: // Analyzer, AnalyzerFactory - + if (simpleName.indexOf("Analyzer") == -1) { simpleName += "Analyzer"; } - + if (simpleName.indexOf("Factory") == -1) { simpleName += "Factory"; } @@ -798,33 +327,39 @@ public static Class getFactoryClass(String simpleName) { // Ignore } } - + + for (AnalyzerFactory factory : this.pluginFramework.getAnalyzersInfo().factories) { + if (factory.getClass().getSimpleName().equals(simpleName)) { + factoryClass = factory.getClass(); + break; + } + } + return factoryClass; } - + /** * Find a {@code FileAnalyzerFactory} which is an instance of the specified * class. If one doesn't exist, create one and register it. * * @param factoryClass the factory class * @return a file analyzer factory - * - * @throws ClassCastException if the class is not a subclass of {@code - * FileAnalyzerFactory} - * @throws IllegalAccessException if the constructor cannot be accessed - * @throws InstantiationException if the class cannot be instantiated - * @throws NoSuchMethodException if no-argument constructor could not be found + * @throws ClassCastException if the class is not a subclass of {@code + * FileAnalyzerFactory} + * @throws IllegalAccessException if the constructor cannot be accessed + * @throws InstantiationException if the class cannot be instantiated + * @throws NoSuchMethodException if no-argument constructor could not be found * @throws InvocationTargetException if the underlying constructor throws an exception */ - private static AnalyzerFactory findFactory(Class factoryClass) + private AnalyzerFactory findFactory(Class factoryClass) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { - for (AnalyzerFactory f : factories) { + for (AnalyzerFactory f : this.pluginFramework.getAnalyzersInfo().factories) { if (f.getClass() == factoryClass) { return f; } } AnalyzerFactory f = (AnalyzerFactory) factoryClass.getDeclaredConstructor().newInstance(); - registerAnalyzer(f); + pluginFramework.registerAnalyzer(f); return f; } @@ -832,16 +367,15 @@ private static AnalyzerFactory findFactory(Class factoryClass) * Finds a suitable analyser class for file name. If the analyzer cannot be * determined by the file extension, try to look at the data in the * InputStream to find a suitable analyzer. - * + *

* Use if you just want to find file type. * - * - * @param in The input stream containing the data + * @param in The input stream containing the data * @param file The file name to get the analyzer for * @return the analyzer factory to use - * @throws java.io.IOException If a problem occurs while reading the data + * @throws IOException If a problem occurs while reading the data */ - public static AnalyzerFactory find(InputStream in, String file) + public AnalyzerFactory find(InputStream in, String file) throws IOException { AnalyzerFactory factory = find(file); // TODO above is not that great, since if 2 analyzers share one extension @@ -860,7 +394,7 @@ public static AnalyzerFactory find(InputStream in, String file) * @param file The file name to get the analyzer for * @return the analyzer factory to use */ - public static AnalyzerFactory find(String file) { + public AnalyzerFactory find(String file) { String path = file; int i; @@ -876,12 +410,12 @@ public static AnalyzerFactory find(String file) { // Try matching the prefix. if (dotpos > 0) { - factory = pre.get(path.substring(0, dotpos).toUpperCase(Locale.ROOT)); + factory = this.pluginFramework.getAnalyzersInfo().prefixes.get(path.substring(0, dotpos).toUpperCase(Locale.ROOT)); if (factory != null) { if (LOGGER.isLoggable(Level.FINER)) { LOGGER.log(Level.FINER, "{0}: chosen by prefix: {1}", - new Object[]{file, - factory.getClass().getSimpleName() }); + new Object[]{file, + factory.getClass().getSimpleName()}); } return factory; } @@ -890,19 +424,19 @@ public static AnalyzerFactory find(String file) { // Now try matching the suffix. We kind of consider this order (first // prefix then suffix) to be workable although for sure there can be // cases when this does not work. - factory = ext.get(path.substring(dotpos + 1).toUpperCase(Locale.ROOT)); + factory = this.pluginFramework.getAnalyzersInfo().extensions.get(path.substring(dotpos + 1).toUpperCase(Locale.ROOT)); if (factory != null) { if (LOGGER.isLoggable(Level.FINER)) { LOGGER.log(Level.FINER, "{0}: chosen by suffix: {1}", - new Object[]{file, - factory.getClass().getSimpleName() }); + new Object[]{file, + factory.getClass().getSimpleName()}); } return factory; } } // file doesn't have any of the prefix or extensions we know, try full match - return FILE_NAMES.get(path.toUpperCase(Locale.ROOT)); + return this.pluginFramework.getAnalyzersInfo().fileNames.get(path.toUpperCase(Locale.ROOT)); } /** @@ -910,10 +444,10 @@ public static AnalyzerFactory find(String file) { * * @param in The stream containing the data to analyze * @return the analyzer factory to use - * @throws java.io.IOException if an error occurs while reading data from - * the stream + * @throws IOException if an error occurs while reading data from + * the stream */ - public static AnalyzerFactory find(InputStream in) throws IOException { + public AnalyzerFactory find(InputStream in) throws IOException { return findForStream(in, ""); } @@ -921,14 +455,14 @@ public static AnalyzerFactory find(InputStream in) throws IOException { * Finds a suitable analyzer class for the data in this stream * corresponding to a file of the specified name * - * @param in The stream containing the data to analyze + * @param in The stream containing the data to analyze * @param file The file name to get the analyzer for * @return the analyzer factory to use - * @throws java.io.IOException if an error occurs while reading data from - * the stream + * @throws IOException if an error occurs while reading data from + * the stream */ - private static AnalyzerFactory findForStream(InputStream in, - String file) throws IOException { + private AnalyzerFactory findForStream(InputStream in, + String file) throws IOException { in.mark(MAGIC_BYTES_NUM); byte[] content = new byte[MAGIC_BYTES_NUM]; @@ -948,14 +482,14 @@ private static AnalyzerFactory findForStream(InputStream in, AnalyzerFactory fac; // First, do precise-magic Matcher matching - for (FileAnalyzerFactory.Matcher matcher : matchers) { + for (FileAnalyzerFactory.Matcher matcher : this.pluginFramework.getAnalyzersInfo().matchers) { if (matcher.getIsPreciseMagic()) { fac = matcher.isMagic(content, in); if (fac != null) { if (LOGGER.isLoggable(Level.FINER)) { LOGGER.log(Level.FINER, - "{0}: chosen by precise magic: {1}", new Object[]{ - file, fac.getClass().getSimpleName() }); + "{0}: chosen by precise magic: {1}", new Object[]{ + file, fac.getClass().getSimpleName()}); } return fac; } @@ -970,15 +504,15 @@ private static AnalyzerFactory findForStream(InputStream in, } // Last, do imprecise-magic Matcher matching - for (FileAnalyzerFactory.Matcher matcher : matchers) { + for (FileAnalyzerFactory.Matcher matcher : this.pluginFramework.getAnalyzersInfo().matchers) { if (!matcher.getIsPreciseMagic()) { fac = matcher.isMagic(content, in); if (fac != null) { if (LOGGER.isLoggable(Level.FINER)) { LOGGER.log(Level.FINER, - "{0}: chosen by imprecise magic: {1}", - new Object[]{file, - fac.getClass().getSimpleName() }); + "{0}: chosen by imprecise magic: {1}", + new Object[]{file, + fac.getClass().getSimpleName()}); } return fac; } @@ -988,44 +522,44 @@ private static AnalyzerFactory findForStream(InputStream in, return null; } - private static AnalyzerFactory findMagicString(String opening, - String file) + private AnalyzerFactory findMagicString(String opening, + String file) throws IOException { // first, try to look up two words in magics String fragment = getWords(opening, 2); - AnalyzerFactory fac = magics.get(fragment); + AnalyzerFactory fac = this.pluginFramework.getAnalyzersInfo().magics.get(fragment); if (fac != null) { if (LOGGER.isLoggable(Level.FINER)) { LOGGER.log(Level.FINER, "{0}: chosen by magic {2}: {1}", - new Object[]{file, fac.getClass().getSimpleName(), - fragment}); + new Object[]{file, fac.getClass().getSimpleName(), + fragment}); } return fac; } // second, try to look up one word in magics fragment = getWords(opening, 1); - fac = magics.get(fragment); + fac = this.pluginFramework.getAnalyzersInfo().magics.get(fragment); if (fac != null) { if (LOGGER.isLoggable(Level.FINER)) { LOGGER.log(Level.FINER, "{0}: chosen by magic {2}: {1}", - new Object[]{file, fac.getClass().getSimpleName(), - fragment}); + new Object[]{file, fac.getClass().getSimpleName(), + fragment}); } return fac; } // try to match initial substrings (DESC strlen) for (Map.Entry entry : - magics.entrySet()) { + this.pluginFramework.getAnalyzersInfo().magics.entrySet()) { String magic = entry.getKey(); if (opening.startsWith(magic)) { fac = entry.getValue(); if (LOGGER.isLoggable(Level.FINER)) { LOGGER.log(Level.FINER, - "{0}: chosen by magic(substr) {2}: {1}", new Object[]{ - file, fac.getClass().getSimpleName(), magic}); + "{0}: chosen by magic(substr) {2}: {1}", new Object[]{ + file, fac.getClass().getSimpleName(), magic}); } return fac; } @@ -1041,7 +575,7 @@ private static AnalyzerFactory findMagicString(String opening, * ends at each and every space character.) * * @param value The source from which words are cut - * @param n The number of words to try to extract + * @param n The number of words to try to extract * @return The extracted words or "" */ private static String getWords(String value, int n) { @@ -1066,13 +600,13 @@ private static String getWords(String value, int n) { * to the first \n after any non-whitespace. (Hashbang, #!, * openings will have superfluous space removed.) * - * @param in The input stream containing the data + * @param in The input stream containing the data * @param sig The initial sequence of bytes in the input stream * @return The extracted string or "" - * @throws java.io.IOException in case of any read error + * @throws IOException in case of any read error */ private static String readOpening(InputStream in, byte[] sig) - throws IOException { + throws IOException { in.mark(MARK_READ_LIMIT); @@ -1096,12 +630,12 @@ private static String readOpening(InputStream in, byte[] sig) StringBuilder opening = new StringBuilder(); BufferedReader readr = new BufferedReader( - new InputStreamReader(in, encoding), OPENING_MAX_CHARS); + new InputStreamReader(in, encoding), OPENING_MAX_CHARS); while ((r = readr.read()) != -1) { if (++nRead > OPENING_MAX_CHARS) { break; } - char c = (char)r; + char c = (char) r; boolean isWhitespace = Character.isWhitespace(c); if (!sawNonWhitespace) { if (isWhitespace) { @@ -1139,31 +673,244 @@ private static String readOpening(InputStream in, byte[] sig) return opening.toString(); } - private static void addCustomizationKey(String k) { - CUSTOMIZATION_KEYS.add(k); - Object[] keys = CUSTOMIZATION_KEYS.toArray(); - customizationHashCode = Objects.hash(keys); + /** + * Get the default Analyzer. + * + * @return default FileAnalyzer + */ + public static AbstractAnalyzer getAnalyzer() { + return AnalyzerFramework.DEFAULT_ANALYZER_FACTORY.getAnalyzer(); } - private static boolean factoriesDifferent(AnalyzerFactory a, - AnalyzerFactory b) { - String a_name = null; - if (a != null) { - a_name = a.getName(); - if (a_name == null) { - a_name = a.getClass().getSimpleName(); + /** + * Gets an analyzer for the specified {@code fileTypeName} if it accords + * with a known {@link FileAnalyzer#getFileTypeName()}. + * + * @param fileTypeName a defined name + * @return a defined instance if known or otherwise {@code null} + */ + public AbstractAnalyzer getAnalyzer(String fileTypeName) { + AnalyzerFactory factory = findByFileTypeName(fileTypeName); + return factory == null ? null : factory.getAnalyzer(); + } + + /** + * Get an analyzer suited to analyze a file. This function will reuse + * analyzers since they are costly. + * + * @param in Input stream containing data to be analyzed + * @param file Name of the file to be analyzed + * @return An analyzer suited for that file content + * @throws java.io.IOException If an error occurs while accessing the data + * in the input stream. + */ + public AbstractAnalyzer getAnalyzer(InputStream in, String file) throws IOException { + AnalyzerFactory factory = find(in, file); + if (factory == null) { + AbstractAnalyzer defaultAnalyzer = getAnalyzer(); + if (LOGGER.isLoggable(Level.FINE)) { + LOGGER.log(Level.FINE, "{0}: fallback {1}", + new Object[]{file, + defaultAnalyzer.getClass().getSimpleName()}); } + return defaultAnalyzer; } - String b_name = null; - if (b != null) { - b_name = b.getName(); - if (b_name == null) { - b_name = b.getClass().getSimpleName(); + return factory.getAnalyzer(); + } + + /** + * Free resources associated with all registered analyzers. + */ + public void returnAnalyzers() { + pluginFramework.returnAnalyzers(); + } + + /** + * Populate a Lucene document with the required fields. + * + * @param doc The document to populate + * @param file The file to index + * @param path Where the file is located (from source root) + * @param fa The analyzer to use on the file + * @param xrefOut Where to write the xref (possibly {@code null}) + * @throws IOException If an exception occurs while collecting the data + * @throws InterruptedException if a timeout occurs + * @throws ForbiddenSymlinkException if symbolic-link checking encounters + * an ineligible link + */ + public void populateDocument(Document doc, File file, String path, + AbstractAnalyzer fa, Writer xrefOut) throws IOException, + InterruptedException, ForbiddenSymlinkException { + + String date = DateTools.timeToString(file.lastModified(), + DateTools.Resolution.MILLISECOND); + path = Util.fixPathIfWindows(path); + doc.add(new Field(QueryBuilder.U, Util.path2uid(path, date), + string_ft_stored_nanalyzed_norms)); + doc.add(new Field(QueryBuilder.FULLPATH, file.getAbsolutePath(), + string_ft_nstored_nanalyzed_norms)); + doc.add(new SortedDocValuesField(QueryBuilder.FULLPATH, + new BytesRef(file.getAbsolutePath()))); + + if (RuntimeEnvironment.getInstance().isHistoryEnabled()) { + try { + HistoryReader hr = HistoryGuru.getInstance().getHistoryReader(file); + if (hr != null) { + doc.add(new TextField(QueryBuilder.HIST, hr)); + // date = hr.getLastCommentDate() //RFE + } + } catch (HistoryException e) { + LOGGER.log(Level.WARNING, "An error occurred while reading history: ", e); } } - if (a_name == null && b_name == null) { - return false; + doc.add(new Field(QueryBuilder.DATE, date, string_ft_stored_nanalyzed_norms)); + doc.add(new SortedDocValuesField(QueryBuilder.DATE, new BytesRef(date))); + + // `path' is not null, as it was passed to Util.path2uid() above. + doc.add(new TextField(QueryBuilder.PATH, path, Store.YES)); + Project project = Project.getProject(path); + if (project != null) { + doc.add(new TextField(QueryBuilder.PROJECT, project.getPath(), Store.YES)); + } + + /* + * Use the parent of the path -- not the absolute file as is done for + * FULLPATH -- so that DIRPATH is the same convention as for PATH + * above. A StringField, however, is used instead of a TextField. + */ + File fpath = new File(path); + String fileParent = fpath.getParent(); + if (fileParent != null && fileParent.length() > 0) { + String normalizedPath = QueryBuilder.normalizeDirPath(fileParent); + StringField npstring = new StringField(QueryBuilder.DIRPATH, + normalizedPath, Store.NO); + doc.add(npstring); + } + + if (fa != null) { + AbstractAnalyzer.Genre g = fa.getGenre(); + if (g == AbstractAnalyzer.Genre.PLAIN || g == AbstractAnalyzer.Genre.XREFABLE || g == AbstractAnalyzer.Genre.HTML) { + doc.add(new Field(QueryBuilder.T, g.typeName(), string_ft_stored_nanalyzed_norms)); + } + fa.analyze(doc, StreamSource.fromFile(file), xrefOut); + + String type = fa.getFileTypeName(); + doc.add(new StringField(QueryBuilder.TYPE, type, Store.YES)); + } + } + + /** + * Get the content type for a named file. + * + * @param in The input stream we want to get the content type for (if we + * cannot determine the content type by the filename) + * @param file The name of the file + * @return The contentType suitable for printing to + * response.setContentType() or null if the factory was not found + * @throws java.io.IOException If an error occurs while accessing the input + * stream. + */ + public String getContentType(InputStream in, String file) throws IOException { + AnalyzerFactory factory = find(in, file); + String type = null; + if (factory != null) { + type = factory.getContentType(); + } + return type; + } + + /** + * Write a browse-able version of the file + * + * @param factory The analyzer factory for this file type + * @param in The input stream containing the data + * @param out Where to write the result + * @param defs definitions for the source file, if available + * @param annotation Annotation information for the file + * @param project Project the file belongs to + * @throws java.io.IOException If an error occurs while creating the output + */ + public static void writeXref(AnalyzerFactory factory, Reader in, + Writer out, Definitions defs, + Annotation annotation, Project project) + throws IOException { + Reader input = in; + if (factory.getGenre() == AbstractAnalyzer.Genre.PLAIN) { + // This is some kind of text file, so we need to expand tabs to + // spaces to match the project's tab settings. + input = ExpandTabsReader.wrap(in, project); + } + + WriteXrefArgs args = new WriteXrefArgs(input, out); + args.setDefs(defs); + args.setAnnotation(annotation); + args.setProject(project); + + AbstractAnalyzer analyzer = factory.getAnalyzer(); + RuntimeEnvironment env = RuntimeEnvironment.getInstance(); + analyzer.setScopesEnabled(env.isScopesEnabled()); + analyzer.setFoldingEnabled(env.isFoldingEnabled()); + analyzer.writeXref(args); + } + + /** + * Writes a browse-able version of the file transformed for immediate + * serving to a web client. + * + * @param contextPath the web context path for + * {@link Util#dumpXref(java.io.Writer, java.io.Reader, java.lang.String)} + * @param factory the analyzer factory for this file type + * @param in the input stream containing the data + * @param out a defined instance to write + * @param defs definitions for the source file, if available + * @param annotation annotation information for the file + * @param project project the file belongs to + * @throws java.io.IOException if an error occurs while creating the output + */ + public static void writeDumpedXref(String contextPath, + AnalyzerFactory factory, Reader in, Writer out, + Definitions defs, Annotation annotation, Project project) + throws IOException { + + File xrefTemp = File.createTempFile("ogxref", ".html"); + try { + try (FileWriter tmpout = new FileWriter(xrefTemp)) { + writeXref(factory, in, tmpout, defs, annotation, project); + } + Util.dumpXref(out, xrefTemp, false, contextPath); + } finally { + xrefTemp.delete(); } - return a_name == null || b_name == null || !a_name.equals(b_name); + } + + /** + * Register the given prefix for the analyzer factory. + * + * @param prefix the prefix + * @param factory the factory + * @see AnalyzerFramework#addPrefix(String, AnalyzerFactory) + */ + public void addPrefix(String prefix, AnalyzerFactory factory) { + this.pluginFramework.addPrefix(prefix, factory); + } + + /** + * Register the given extension for the analyzer factory. + * + * @param extension the extension + * @param factory the factory + * @see AnalyzerFramework#addExtension(String, AnalyzerFactory) (String, AnalyzerFactory) + */ + public void addExtension(String extension, AnalyzerFactory factory) { + this.pluginFramework.addExtension(extension, factory); + } + + /** + * Return the instance of analyzer plugin framework. + * + * @return the instance of analyzer plugin framework + */ + public AnalyzerFramework getPluginFramework() { + return pluginFramework; } } diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuruHelp.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuruHelp.java index fe848d5248b..5bee667f78f 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuruHelp.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzerGuruHelp.java @@ -40,35 +40,36 @@ public class AnalyzerGuruHelp { * {@link AnalyzerGuru#getMagicsMap()}, and * {@link AnalyzerGuru#getAnalyzerFactoryMatchers()}. * @return a defined, multi-line String + * @param analyzerGuru an instance of analyzer guru with info about analyzers */ - public static String getUsage() { + public static String getUsage(AnalyzerGuru analyzerGuru) { StringBuilder b = new StringBuilder(); b.append("AnalyzerGuru prefixes:\n"); - byKey(AnalyzerGuru.getPrefixesMap()).forEach((kv) -> { + byKey(analyzerGuru.getPrefixesMap()).forEach((kv) -> { b.append(String.format("%-10s : %s\n", reportable(kv.key + '*'), reportable(kv.fac))); }); b.append("\nAnalyzerGuru extensions:\n"); - byKey(AnalyzerGuru.getExtensionsMap()).forEach((kv) -> { + byKey(analyzerGuru.getExtensionsMap()).forEach((kv) -> { b.append(String.format("*.%-7s : %s\n", reportable(kv.key.toLowerCase(Locale.ROOT)), reportable(kv.fac))); }); b.append("\nAnalyzerGuru magic strings:\n"); - byFactory(AnalyzerGuru.getMagicsMap()).forEach((kv) -> { + byFactory(analyzerGuru.getMagicsMap()).forEach((kv) -> { b.append(String.format("%-23s : %s\n", reportable(kv.key), reportable(kv.fac))); }); b.append("\nAnalyzerGuru magic matchers:\n"); - AnalyzerGuru.getAnalyzerFactoryMatchers().forEach((m) -> { + analyzerGuru.getAnalyzerFactoryMatchers().forEach((m) -> { if (m.getIsPreciseMagic()) { b.append(reportable(m)); } }); - AnalyzerGuru.getAnalyzerFactoryMatchers().forEach((m) -> { + analyzerGuru.getAnalyzerFactoryMatchers().forEach((m) -> { if (!m.getIsPreciseMagic()) { b.append(reportable(m)); } diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzersInfo.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzersInfo.java new file mode 100644 index 00000000000..0d389284130 --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/AnalyzersInfo.java @@ -0,0 +1,180 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + */ +package org.opengrok.indexer.analysis; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.SortedMap; +import java.util.SortedSet; +import java.util.TreeMap; +import java.util.TreeSet; + +/** + * A class wrapping information about used analyzers. + */ +public class AnalyzersInfo { + + /** + * Descending string length comparator for magics + */ + private static final Comparator DESCENDING_LENGTH_COMPARATOR = + Comparator.comparingInt(String::length).thenComparing(String::toString); + + /** + * Modified when + * {@link AnalyzerFramework#addExtension(String, AnalyzerFactory)} + * or + * {@link AnalyzerFramework#addPrefix(String, AnalyzerFactory)} + * are called to augment the value in {@link AnalyzerGuru#getVersionNo()}. + */ + public final SortedSet customizations; + + /** + * Map from file names to analyzer factories. + */ + public final Map fileNames; + + /** + * Map from file extensions to analyzer factories. + */ + public final Map extensions; + + /** + * Map from file prefixes to analyzer factories. + */ + public final Map prefixes; + + /** + * Map from magic strings to analyzer factories. + */ + public final SortedMap magics; + + /** + * List of matcher objects which can be used to determine which analyzer + * factory to use. + */ + public final List matchers; + + /** + * List of all registered {@code FileAnalyzerFactory} instances. + */ + public final List factories; + + /** + * A map of {@link FileAnalyzer#getFileTypeName()} to human readable analyzer name. + */ + public final Map fileTypeDescriptions; + + /** + * Maps from {@link FileAnalyzer#getFileTypeName()} to + * {@link FileAnalyzerFactory} + */ + public final Map filetypeFactories; + + /** + * Maps from {@link FileAnalyzer#getFileTypeName()} to + * {@link FileAnalyzer#getVersionNo()} + */ + public final Map analyzerVersions; + + + /** + * Construct an empty analyzers info. + */ + public AnalyzersInfo() { + this( + new TreeSet<>(), + new HashMap<>(), + new HashMap<>(), + new HashMap<>(), + new TreeMap<>(DESCENDING_LENGTH_COMPARATOR), + new ArrayList<>(), + new ArrayList<>(), + new HashMap<>(), + new HashMap<>(), + new HashMap<>() + ); + } + + /** + * Construct the analyzers info with given collections. + * + * @param customizations the customization keys set + * @param fileNames map of filenames to analyzers factories + * @param extensions map of extensions to analyzers factories + * @param prefixes map of prefixes to analyzers factories + * @param magics map of magics to analyzers factories + * @param matchers a list of matchers for analyzers factories + * @param factories a list of analyzers factories + * @param fileTypeDescriptions map of analyzer names to analyzer descriptions + * @param filetypeFactories map of file type to analyzers factories + * @param analyzerVersions map of analyzer names to analyzer versions + */ + private AnalyzersInfo( + SortedSet customizations, + Map fileNames, + Map extensions, + Map prefixes, + SortedMap magics, + List matchers, + List factories, + Map fileTypeDescriptions, + Map filetypeFactories, + Map analyzerVersions + ) { + this.customizations = customizations; + this.fileNames = fileNames; + this.extensions = extensions; + this.prefixes = prefixes; + this.magics = magics; + this.matchers = matchers; + this.factories = factories; + this.fileTypeDescriptions = fileTypeDescriptions; + this.filetypeFactories = filetypeFactories; + this.analyzerVersions = analyzerVersions; + } + + /** + * Make the object unmodifiable. + * + * @return a new object reference with all properties wrapped as unmodifiable collections + */ + public AnalyzersInfo freeze() { + return new AnalyzersInfo( + Collections.unmodifiableSortedSet(this.customizations), + Collections.unmodifiableMap(this.fileNames), + Collections.unmodifiableMap(this.extensions), + Collections.unmodifiableMap(this.prefixes), + Collections.unmodifiableSortedMap(this.magics), + Collections.unmodifiableList(this.matchers), + Collections.unmodifiableList(this.factories), + Collections.unmodifiableMap(this.fileTypeDescriptions), + Collections.unmodifiableMap(this.filetypeFactories), + Collections.unmodifiableMap(this.analyzerVersions) + ); + } +} diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/IAnalyzerPlugin.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/IAnalyzerPlugin.java new file mode 100644 index 00000000000..e5f2e6352f6 --- /dev/null +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/IAnalyzerPlugin.java @@ -0,0 +1,28 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + */ +package org.opengrok.indexer.analysis; + +public interface IAnalyzerPlugin { + + AnalyzerFactory getFactory(); +} diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/archive/BZip2Analyzer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/archive/BZip2Analyzer.java index 1ba6dfc4cea..94e1c16e001 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/archive/BZip2Analyzer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/archive/BZip2Analyzer.java @@ -35,6 +35,7 @@ import org.opengrok.indexer.analysis.AnalyzerGuru; import org.opengrok.indexer.analysis.FileAnalyzer; import org.opengrok.indexer.analysis.StreamSource; +import org.opengrok.indexer.configuration.RuntimeEnvironment; /** * Analyzes a BZip2 file Created on September 22, 2005 @@ -80,7 +81,7 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut) String newname = path.substring(0, path.lastIndexOf('.')); //System.err.println("BZIPPED OF = " + newname); try (InputStream in = bzSrc.getStream()) { - fa = AnalyzerGuru.getAnalyzer(in, newname); + fa = RuntimeEnvironment.getInstance().getAnalyzerGuru().getAnalyzer(in, newname); } if (!(fa instanceof BZip2Analyzer)) { if (fa.getGenre() == Genre.PLAIN || fa.getGenre() == Genre.XREFABLE) { diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/archive/GZIPAnalyzer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/archive/GZIPAnalyzer.java index ab2ee17cc37..b58cec962ba 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/archive/GZIPAnalyzer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/archive/GZIPAnalyzer.java @@ -38,6 +38,7 @@ import org.opengrok.indexer.analysis.AnalyzerGuru; import org.opengrok.indexer.analysis.FileAnalyzer; import org.opengrok.indexer.analysis.StreamSource; +import org.opengrok.indexer.configuration.RuntimeEnvironment; import org.opengrok.indexer.logger.LoggerFactory; /** @@ -85,7 +86,7 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut) String newname = path.substring(0, path.length() - 3); //System.err.println("GZIPPED OF = " + newname); try (InputStream gzis = gzSrc.getStream()) { - fa = AnalyzerGuru.getAnalyzer(gzis, newname); + fa = RuntimeEnvironment.getInstance().getAnalyzerGuru().getAnalyzer(gzis, newname); } if (fa == null) { this.g = Genre.DATA; diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/executables/JarAnalyzer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/executables/JarAnalyzer.java index 402ef3492cb..c3c03f6fc0b 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/executables/JarAnalyzer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/analysis/executables/JarAnalyzer.java @@ -32,10 +32,10 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; import org.opengrok.indexer.analysis.AnalyzerFactory; -import org.opengrok.indexer.analysis.AnalyzerGuru; import org.opengrok.indexer.analysis.FileAnalyzer; import org.opengrok.indexer.analysis.OGKTextField; import org.opengrok.indexer.analysis.StreamSource; +import org.opengrok.indexer.configuration.RuntimeEnvironment; import org.opengrok.indexer.search.QueryBuilder; import org.opengrok.indexer.web.Util; @@ -82,7 +82,7 @@ public void analyze(Document doc, StreamSource src, Writer xrefOut) throws IOExc fout.write(ename); fout.write("\n"); - AnalyzerFactory fac = AnalyzerGuru.find(ename); + AnalyzerFactory fac = RuntimeEnvironment.getInstance().getAnalyzerGuru().find(ename); if (fac instanceof JavaClassAnalyzerFactory) { if (xrefOut != null) { xrefOut.append("
"); diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/authorization/AuthorizationEntity.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/authorization/AuthorizationEntity.java index 0c7d17461b5..8356b48a99e 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/authorization/AuthorizationEntity.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/authorization/AuthorizationEntity.java @@ -476,7 +476,7 @@ public synchronized void setWorking() { /** * Check if this plugin has failed during loading or is missing. * - * This method has the same effect as !{@link isWorking()}. + * This method has the same effect as !{@link #isWorking()}. * * @return true if failed, true otherwise * @see #isWorking() diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/authorization/AuthorizationFramework.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/authorization/AuthorizationFramework.java index 6826d64e803..d580ad08c07 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/authorization/AuthorizationFramework.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/authorization/AuthorizationFramework.java @@ -44,7 +44,7 @@ * * @author Krystof Tulinger */ -public final class AuthorizationFramework extends PluginFramework { +public final class AuthorizationFramework extends PluginFramework { private static final Logger LOGGER = LoggerFactory.getLogger(AuthorizationFramework.class); @@ -59,11 +59,6 @@ public final class AuthorizationFramework extends PluginFramework * + * @param localStack a local stack used for loading the plugins * @see IAuthorizationPlugin#load(java.util.Map) * @see IAuthorizationPlugin#unload() * @see Configuration#getPluginDirectory() */ @SuppressWarnings({"rawtypes", "unchecked"}) @Override - protected void afterReload() { + protected void afterReload(AuthorizationStack localStack) { + if (getPluginDirectory() == null || !getPluginDirectory().isDirectory() || !getPluginDirectory().canRead()) { + LOGGER.log(Level.WARNING, "All requests allowed."); + return; + } + if (stack == null) { LOGGER.log(Level.WARNING, "Plugin stack not found in configuration: null. All requests allowed."); return; } // fire load events - loadAllPlugins(loadingStack); + loadAllPlugins(localStack); AuthorizationStack oldStack; /** @@ -367,7 +380,7 @@ protected void afterReload() { lock.writeLock().lock(); try { oldStack = stack; - stack = loadingStack; + stack = localStack; // increase the current plugin version tracked by the framework increasePluginVersion(); @@ -381,7 +394,6 @@ protected void afterReload() { // clean the old stack removeAll(oldStack); oldStack = null; - loadingStack = null; } /** diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java index 1595dfc205d..8d95c4e3a33 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/configuration/RuntimeEnvironment.java @@ -57,6 +57,7 @@ import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.opengrok.indexer.analysis.AnalyzerGuru; import org.opengrok.indexer.authorization.AuthorizationFramework; import org.opengrok.indexer.authorization.AuthorizationStack; import org.opengrok.indexer.history.HistoryGuru; @@ -135,11 +136,16 @@ private RuntimeEnvironment() { /** Instance of authorization framework.*/ private AuthorizationFramework authFramework; - /** Gets the thread pool used for multi-project searches. */ + private AnalyzerGuru analyzerGuru; + + /** + * Gets the thread pool used for multi-project searches. + */ public ExecutorService getSearchExecutor() { return lzSearchExecutor.get(); } + private ExecutorService newSearchExecutor() { return Executors.newFixedThreadPool( this.getMaxSearchThreadCount(), @@ -300,7 +306,7 @@ public int getInteractiveCommandTimeout() { public void setInteractiveCommandTimeout(int timeout) { setConfigurationValue("interactiveCommandTimeout", timeout); } - + public Statistics getStatistics() { return statistics; } @@ -706,7 +712,7 @@ public List getRepositories() { public void setRepositories(List repositories) { setConfigurationValue("repositories", repositories); } - + public void removeRepositories() { try { configLock.writeLock().lock(); @@ -715,7 +721,7 @@ public void removeRepositories() { configLock.writeLock().unlock(); } } - + /** * Search through the directory for repositories and use the result to replace * the lists of repositories in both RuntimeEnvironment/Configuration and HistoryGuru. @@ -1591,6 +1597,18 @@ public synchronized AuthorizationFramework getAuthorizationFramework() { return authFramework; } + /** + * Return the analyzer guru. + * + * @return the analyzer guru instance, never {@code null}. + */ + public synchronized AnalyzerGuru getAnalyzerGuru() { + if (analyzerGuru == null) { + analyzerGuru = new AnalyzerGuru(getPluginDirectory()); + } + return analyzerGuru; + } + /** * Set the authorization framework for this environment. Unload all * previously load plugins. @@ -1671,6 +1689,10 @@ public void applyConfig(Configuration config, boolean reindex, boolean interacti getAuthorizationFramework().setStack(getPluginStack()); getAuthorizationFramework().reload(); + // set the new plugin directory and reload the analyzer framework + getAnalyzerGuru().getPluginFramework().setPluginDirectory(getPluginDirectory()); + getAnalyzerGuru().getPluginFramework().reload(); + messagesContainer.setMessageLimit(getMessageLimit()); } diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/framework/PluginClassLoader.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/framework/PluginClassLoader.java index 172294c59ed..eb46c0926e3 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/framework/PluginClassLoader.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/framework/PluginClassLoader.java @@ -28,8 +28,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.Arrays; -import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.logging.Level; @@ -44,7 +44,7 @@ public class PluginClassLoader extends ClassLoader { @SuppressWarnings("rawtypes") - private final Map cache = new HashMap<>(); + private final Map cache = new ConcurrentHashMap<>(); private static final Logger LOGGER = LoggerFactory.getLogger(PluginClassLoader.class); private static final String[] CLASS_WHITELIST = new String[]{ @@ -232,8 +232,6 @@ public Class loadClass(String name, boolean resolveIt) throws ClassNotFoundExcep return c; } - checkClassname(name); - // find already loaded class if ((c = findLoadedClass(name)) != null) { cache.put(name, c); @@ -257,6 +255,8 @@ public Class loadClass(String name, boolean resolveIt) throws ClassNotFoundExcep } } + checkClassname(name); + try { checkPackage(name); // load it from file diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/framework/PluginFramework.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/framework/PluginFramework.java index e6586a54447..1c8e6d9149e 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/framework/PluginFramework.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/framework/PluginFramework.java @@ -45,7 +45,7 @@ * * @author Krystof Tulinger */ -public abstract class PluginFramework { +public abstract class PluginFramework { private static final Logger LOGGER = LoggerFactory.getLogger(PluginFramework.class); @@ -216,7 +216,8 @@ private PluginType loadClass(String classname) throws ClassNotFoundException, // check for implemented interfaces or extended superclasses for (Class intf1 : getSuperclassesAndInterfaces(c)) { - if (intf1.getCanonicalName().equals(classType.getCanonicalName()) + if (intf1.getCanonicalName() != null && // anonymous classes + intf1.getCanonicalName().equals(classType.getCanonicalName()) && !Modifier.isAbstract(c.getModifiers())) { // call to non-parametric constructor return (PluginType) c.getDeclaredConstructor().newInstance(); @@ -251,11 +252,12 @@ protected List getSuperclassesAndInterfaces(Class clazz) { * delegates the loading to the custom class loader * {@link #loadClass(String)}. * + * @param data a custom data created with {@link #beforeReload()} * @param classfiles list of files which possibly contain a java class * @see #handleLoadClass(String) * @see #loadClass(String) */ - private void loadClassFiles(List classfiles) { + private void loadClassFiles(DataType data, List classfiles) { PluginType plugin; for (File file : classfiles) { @@ -265,7 +267,7 @@ private void loadClassFiles(List classfiles) { } // Load the class in memory and try to find a configured space for this class. if ((plugin = handleLoadClass(classname)) != null) { - classLoaded(plugin); + classLoaded(data, plugin); } } } @@ -277,11 +279,12 @@ private void loadClassFiles(List classfiles) { * delegates the loading to the custom class loader * {@link #loadClass(String)}. * + * @param data a custom data created with {@link #beforeReload()} * @param jarfiles list of jar files containing java classes * @see #handleLoadClass(String) * @see #loadClass(String) */ - private void loadJarFiles(List jarfiles) { + private void loadJarFiles(DataType data, List jarfiles) { PluginType pf; for (File file : jarfiles) { @@ -298,7 +301,7 @@ private void loadJarFiles(List jarfiles) { } // Load the class in memory and try to find a configured space for this class. if ((pf = handleLoadClass(classname)) != null) { - classLoaded(pf); + classLoaded(data, pf); } } } catch (IOException ex) { @@ -326,23 +329,34 @@ private String getClassName(JarEntry f) { * Allow the implementing class to interact with the loaded class when the class * was loaded with the custom class loader. * + * @param data a custom data created with {@link #beforeReload()} * @param plugin the loaded plugin */ - protected abstract void classLoaded(PluginType plugin); + protected abstract void classLoaded(DataType data, PluginType plugin); /** * Perform custom operations before the plugins are loaded. + *

+ * If this function returns {@code null}, the reloading is completely skipped and {@link #afterReload(Object)} + * is never called. + * + * @return the implementor can generate custom data which will be later passed to {@link #afterReload(Object)} */ - protected abstract void beforeReload(); + protected abstract DataType beforeReload(); /** * Perform custom operations when the framework has reloaded all available plugins. *

* When this is invoked, all plugins has been loaded into the memory and for each available plugin - * the {@link #classLoaded(Object)} was invoked. + * the {@link #classLoaded(Object, Object)} was invoked. + *

+ * When {@link #beforeReload()} returns {@code null}, the reloading is skipped, and this method is + * never called. + * + * @param data a custom data created with {@link #beforeReload()} */ - protected abstract void afterReload(); + protected abstract void afterReload(DataType data); /** * Calling this function forces the framework to reload the plugins. @@ -352,30 +366,38 @@ private String getClassName(JarEntry f) { */ @SuppressWarnings({"rawtypes", "unchecked"}) public final void reload() { - if (pluginDirectory == null || !pluginDirectory.isDirectory() || !pluginDirectory.canRead()) { - LOGGER.log(Level.WARNING, "Plugin directory not found or not readable: {0}. " - + "All requests allowed.", pluginDirectory); + // notify the implementing class that the reload is about to begin + final DataType localData = beforeReload(); + + if (localData == null) { + // the implementor does not want to reload the plugins + LOGGER.log(Level.WARNING, "Loading plugins from {0} as instructed by the implementer's class", pluginDirectory); return; } - LOGGER.log(Level.INFO, "Plugins are being reloaded from {0}", pluginDirectory.getAbsolutePath()); + try { + if (pluginDirectory == null || !pluginDirectory.isDirectory() || !pluginDirectory.canRead()) { + LOGGER.log(Level.WARNING, "Plugin directory not found or not readable: {0}.", pluginDirectory); + return; + } + + LOGGER.log(Level.INFO, "Plugins are being reloaded from {0}", pluginDirectory.getAbsolutePath()); - // trashing out the old instance of the loaded enables us - // to reload the stack at runtime - loader = (PluginClassLoader) AccessController.doPrivileged((PrivilegedAction) () -> new PluginClassLoader(pluginDirectory)); + // trashing out the old instance of the loaded enables us + // to reload the stack at runtime + loader = (PluginClassLoader) AccessController.doPrivileged((PrivilegedAction) () -> new PluginClassLoader(pluginDirectory)); - // notify the implementing class that the reload is about to begin - beforeReload(); - // load all other possible plugin classes. - if (isLoadClassesEnabled()) { - loadClassFiles(IOUtils.listFilesRec(pluginDirectory, ".class")); - } - if (isLoadJarsEnabled()) { - loadJarFiles(IOUtils.listFiles(pluginDirectory, ".jar")); + // load all other possible plugin classes. + if (isLoadClassesEnabled()) { + loadClassFiles(localData, IOUtils.listFilesRec(pluginDirectory, ".class")); + } + if (isLoadJarsEnabled()) { + loadJarFiles(localData, IOUtils.listFiles(pluginDirectory, ".jar")); + } + } finally { + // notify the implementing class that the reload has ended + afterReload(localData); } - - // notify the implementing class that the reload has ended - afterReload(); } } diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java index 8837c50d168..02c9be39d13 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/IndexDatabase.java @@ -317,7 +317,7 @@ private void initialize() throws IOException { indexDirectory = FSDirectory.open(indexDir.toPath(), lockfact); ignoredNames = env.getIgnoredNames(); includedNames = env.getIncludedNames(); - analyzerGuru = new AnalyzerGuru(); + analyzerGuru = env.getAnalyzerGuru(); xrefDir = new File(env.getDataRootFile(), XREF_DIR); listeners = new CopyOnWriteArrayList<>(); dirtyFile = new File(indexDir, "dirty"); @@ -804,10 +804,9 @@ private void addFile(File file, String path, Ctags ctags) private AbstractAnalyzer getAnalyzerFor(File file, String path) throws IOException { - AbstractAnalyzer fa; try (InputStream in = new BufferedInputStream( new FileInputStream(file))) { - return AnalyzerGuru.getAnalyzer(in, path); + return analyzerGuru.getAnalyzer(in, path); } } @@ -1710,7 +1709,7 @@ private boolean checkSettings(File file, continue; } - long reqGuruVersion = AnalyzerGuru.getVersionNo(); + long reqGuruVersion = analyzerGuru.getVersionNo(); Long actGuruVersion = settings.getAnalyzerGuruVersion(); /* * For an older OpenGrok index that does not yet have a defined, @@ -1731,8 +1730,7 @@ private boolean checkSettings(File file, break; } - AnalyzerFactory fac = - AnalyzerGuru.findByFileTypeName(fileTypeName); + AnalyzerFactory fac = analyzerGuru.findByFileTypeName(fileTypeName); if (fac != null) { fa = fac.getAnalyzer(); } @@ -1757,7 +1755,7 @@ private boolean checkSettings(File file, } // Verify Analyzer version, or return a value to indicate mismatch. - long reqVersion = AnalyzerGuru.getAnalyzerVersionNo(fileTypeName); + long reqVersion = analyzerGuru.getAnalyzerVersionNo(fileTypeName); Long actVersion = settings.getAnalyzerVersion(fileTypeName); if (actVersion == null || !actVersion.equals(reqVersion)) { if (LOGGER.isLoggable(Level.FINE)) { @@ -1796,8 +1794,8 @@ private void writeAnalysisSettings() throws IOException { settings.setProjectName(project != null ? project.getName() : null); settings.setTabSize(project != null && project.hasTabSizeSetting() ? project.getTabSize() : 0); - settings.setAnalyzerGuruVersion(AnalyzerGuru.getVersionNo()); - settings.setAnalyzersVersions(AnalyzerGuru.getAnalyzersVersionNos()); + settings.setAnalyzerGuruVersion(analyzerGuru.getVersionNo()); + settings.setAnalyzersVersions(analyzerGuru.getAnalyzersVersionNos()); IndexAnalysisSettingsAccessor dao = new IndexAnalysisSettingsAccessor(); dao.write(writer, settings); diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java index 263817af073..e6602303b64 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/index/Indexer.java @@ -149,8 +149,8 @@ public static void main(String[] argv) { PrintStream helpStream = status != 0 ? System.err : System.out; helpStream.println(helpUsage); if (helpDetailed) { - helpStream.println(AnalyzerGuruHelp.getUsage()); - helpStream.println(ConfigurationHelp.getSamples()); + System.err.println(AnalyzerGuruHelp.getUsage(RuntimeEnvironment.getInstance().getAnalyzerGuru())); + System.err.println(ConfigurationHelp.getSamples()); } System.exit(status); } @@ -837,7 +837,7 @@ private static void die(String message) { } private static void configureFileAnalyzer(String fileSpec, String analyzer) { - + final AnalyzerGuru analyzerGuru = env.getAnalyzerGuru(); boolean prefix = false; // removing '.' from file specification @@ -853,20 +853,20 @@ private static void configureFileAnalyzer(String fileSpec, String analyzer) { // Disable analyzer? if (analyzer.equals("-")) { if (prefix) { - AnalyzerGuru.addPrefix(fileSpec, null); + analyzerGuru.addPrefix(fileSpec, null); } else { - AnalyzerGuru.addExtension(fileSpec, null); + analyzerGuru.addExtension(fileSpec, null); } } else { try { if (prefix) { - AnalyzerGuru.addPrefix( + analyzerGuru.addPrefix( fileSpec, - AnalyzerGuru.findFactory(analyzer)); + analyzerGuru.findFactory(analyzer)); } else { - AnalyzerGuru.addExtension( + analyzerGuru.addExtension( fileSpec, - AnalyzerGuru.findFactory(analyzer)); + analyzerGuru.findFactory(analyzer)); } } catch (ClassNotFoundException | IllegalAccessException | InstantiationException | NoSuchMethodException diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/search/context/OGKUnifiedHighlighter.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/search/context/OGKUnifiedHighlighter.java index e8161de6d43..7ceabe9d06d 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/search/context/OGKUnifiedHighlighter.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/search/context/OGKUnifiedHighlighter.java @@ -41,7 +41,6 @@ import org.apache.lucene.search.uhighlight.UnifiedHighlighter; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.CharacterRunAutomaton; -import org.opengrok.indexer.analysis.AnalyzerGuru; import org.opengrok.indexer.analysis.ExpandTabsReader; import org.opengrok.indexer.analysis.StreamSource; import org.opengrok.indexer.configuration.RuntimeEnvironment; @@ -97,7 +96,7 @@ public Analyzer getIndexAnalyzer() { if (ftname == null) { return indexAnalyzer; } - Analyzer fa = AnalyzerGuru.getAnalyzer(ftname); + Analyzer fa = env.getAnalyzerGuru().getAnalyzer(ftname); return fa == null ? indexAnalyzer : fa; } diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/web/PageConfig.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/web/PageConfig.java index a615a805fc9..c70854dacf2 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/web/PageConfig.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/web/PageConfig.java @@ -59,7 +59,6 @@ import javax.ws.rs.core.HttpHeaders; import org.opengrok.indexer.Info; import org.opengrok.indexer.analysis.AbstractAnalyzer; -import org.opengrok.indexer.analysis.AnalyzerGuru; import org.opengrok.indexer.analysis.ExpandTabsReader; import org.opengrok.indexer.authorization.AuthorizationFramework; import org.opengrok.indexer.configuration.Group; @@ -246,7 +245,7 @@ public DiffData getDiffData() { + getUriEncodedPath() + "\">history"; return data; } - data.genre = AnalyzerGuru.getGenre(getResourceFile().getName()); + data.genre = env.getAnalyzerGuru().getGenre(getResourceFile().getName()); if (data.genre == null || txtGenres.contains(data.genre)) { InputStream[] in = new InputStream[2]; @@ -270,7 +269,7 @@ public DiffData getDiffData() { */ for (int i = 0; i < 2 && data.genre == null; i++) { try { - data.genre = AnalyzerGuru.getGenre(in[i]); + data.genre = env.getAnalyzerGuru().getGenre(in[i]); } catch (IOException e) { data.errorMsg = "Unable to determine the file type: " + Util.htmlize(e.getMessage()); diff --git a/opengrok-indexer/src/main/java/org/opengrok/indexer/web/SearchHelper.java b/opengrok-indexer/src/main/java/org/opengrok/indexer/web/SearchHelper.java index fea6c1d83b4..4e320fa09b4 100644 --- a/opengrok-indexer/src/main/java/org/opengrok/indexer/web/SearchHelper.java +++ b/opengrok-indexer/src/main/java/org/opengrok/indexer/web/SearchHelper.java @@ -56,7 +56,6 @@ import org.apache.lucene.search.spell.SuggestMode; import org.apache.lucene.search.spell.SuggestWord; import org.apache.lucene.store.FSDirectory; -import org.opengrok.indexer.analysis.AnalyzerGuru; import org.opengrok.indexer.analysis.CompatibleAnalyser; import org.opengrok.indexer.analysis.Definitions; import org.opengrok.indexer.configuration.Project; @@ -228,7 +227,7 @@ public class SearchHelper { * @return Set of tuples with file type and description. */ public static Set> getFileTypeDescriptions() { - return AnalyzerGuru.getfileTypeDescriptions().entrySet(); + return RuntimeEnvironment.getInstance().getAnalyzerGuru().getfileTypeDescriptions().entrySet(); } File indexDir; diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/AnalyzerFrameworkTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/AnalyzerFrameworkTest.java new file mode 100644 index 00000000000..7e7f040d385 --- /dev/null +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/AnalyzerFrameworkTest.java @@ -0,0 +1,127 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * See LICENSE.txt included in this distribution for the specific + * language governing permissions and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at LICENSE.txt. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (c) 2019 Oracle and/or its affiliates. All rights reserved. + */ +package org.opengrok.indexer.analysis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.File; +import java.io.IOException; +import java.io.Reader; +import java.io.StringWriter; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import org.junit.Assert; +import org.junit.Test; +import org.opengrok.indexer.configuration.RuntimeEnvironment; + +public class AnalyzerFrameworkTest { + + private final File pluginDirectory; + + public AnalyzerFrameworkTest() throws URISyntaxException { + pluginDirectory = Paths.get(getClass().getResource("/analysis/plugins/testplugins.jar").toURI()).toFile().getParentFile(); + } + + /** + * Analyzer framework should be started reloaded from the plugin directory. + */ + @Test + public void testReloadDefault() { + AnalyzerFramework framework = new AnalyzerFramework(null); + + Assert.assertFalse("Analyzer framework should load the default analyzers", framework.getAnalyzersInfo().factories.isEmpty()); + Assert.assertFalse("Analyzer framework should load the default analyzers", framework.getAnalyzersInfo().extensions.isEmpty()); + Assert.assertFalse("Analyzer framework should load the default analyzers", framework.getAnalyzersInfo().magics.isEmpty()); + Assert.assertFalse("Analyzer framework should load the default analyzers", framework.getAnalyzersInfo().prefixes.isEmpty()); + Assert.assertFalse("Analyzer framework should load the default analyzers", framework.getAnalyzersInfo().matchers.isEmpty()); + } + + /** + * Analyzer framework should be started reloaded from the plugin directory. + */ + @Test + public void testReloadSimple() { + AnalyzerFramework framework = new AnalyzerFramework(pluginDirectory.getPath()); + framework.setLoadClasses(false); // to avoid noise when loading classes of other tests + + // Ensure the framework was setup correctly. + assertNotNull(framework.getPluginDirectory()); + assertEquals(pluginDirectory, framework.getPluginDirectory()); + + final List factories = framework.getAnalyzersInfo().factories; + Assert.assertTrue(factories.stream().map(f -> f.getClass().getSimpleName()).anyMatch(name -> "DummyAnalyzerFactory".equals(name))); + Assert.assertTrue(factories.stream().map(AnalyzerFactory::getName).anyMatch(name -> "Dummy".equals(name))); + Assert.assertTrue(framework.getAnalyzersInfo().extensions.containsKey("DUMMY")); + } + + @Test + public void testGuruRegisteredExtension() { + final RuntimeEnvironment env = RuntimeEnvironment.getInstance(); + env.setPluginDirectory(pluginDirectory.getAbsolutePath()); + final AnalyzerGuru guru = env.getAnalyzerGuru(); + + Assert.assertNotNull(guru.find("file.dummy")); + Assert.assertEquals("DummyAnalyzerFactory", guru.find("file.dummy").getClass().getSimpleName()); + Assert.assertEquals("Dummy", guru.find("file.dummy").getName()); + } + + @Test + public void testAnalyzeStream() throws IOException { + final Path file = Files.createTempFile("opengrok", "analyzer-framework-test.dummy"); + final List lines = Arrays.asList(new String[]{ + "Some random content", + "\tsecond line is padded", + "james.bond@italy.com", + "opengrok is the best" + }); + + Files.write(file, lines); + + try (Reader reader = Files.newBufferedReader(file); StringWriter out = new StringWriter()) { + final RuntimeEnvironment env = RuntimeEnvironment.getInstance(); + env.setPluginDirectory(pluginDirectory.getAbsolutePath()); + final AnalyzerGuru guru = env.getAnalyzerGuru(); + + final AnalyzerFactory factory = guru.find(file.toString()); + final AbstractAnalyzer analyzer = factory.getAnalyzer(); + + Assert.assertEquals(AbstractAnalyzer.Genre.PLAIN, analyzer.getGenre()); + + final Xrefer xrefer = analyzer.writeXref(new WriteXrefArgs(reader, out)); + + Assert.assertEquals(4, xrefer.getLineNumber()); + Assert.assertEquals(4, xrefer.getLOC()); + + // assert content + Assert.assertEquals(lines.stream().collect(Collectors.joining("\n")), out.toString()); + } finally { + Files.deleteIfExists(file); + } + } +} diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/AnalyzerGuruHelpTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/AnalyzerGuruHelpTest.java index 2cbc82391c2..9d7c4882175 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/AnalyzerGuruHelpTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/AnalyzerGuruHelpTest.java @@ -25,6 +25,7 @@ import static org.junit.Assert.assertTrue; import org.junit.Test; +import org.opengrok.indexer.configuration.RuntimeEnvironment; /** * Represents a container for tests of {@link AnalyzerGuruHelp}. @@ -32,7 +33,7 @@ public class AnalyzerGuruHelpTest { @Test public void shouldCreateReadableUsage() { - String usage = AnalyzerGuruHelp.getUsage(); + String usage = AnalyzerGuruHelp.getUsage(RuntimeEnvironment.getInstance().getAnalyzerGuru()); assertTrue("usage is not empty", !usage.isEmpty()); assertTrue("usage contains \"*.\"", usage.contains("*.")); assertTrue("usage contains \"#!\"", usage.contains("#!")); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/AnalyzerGuruTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/AnalyzerGuruTest.java index 93cd8d3a807..34e1361ebf9 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/AnalyzerGuruTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/AnalyzerGuruTest.java @@ -57,10 +57,10 @@ public class AnalyzerGuruTest { @Test public void testGetFileTypeDescriptions() { - Map map = AnalyzerGuru.getfileTypeDescriptions(); + Map map = new AnalyzerGuru().getfileTypeDescriptions(); Assert.assertTrue(map.size() > 0); } - + /** * Test that we get the correct analyzer if the file name exactly matches a * known extension. @@ -70,7 +70,7 @@ public void testFileNameSameAsExtension() throws Exception { ByteArrayInputStream in = new ByteArrayInputStream( "#!/bin/sh\nexec /usr/bin/zip \"$@\"\n".getBytes("US-ASCII")); String file = "/dummy/path/to/source/zip"; - AbstractAnalyzer fa = AnalyzerGuru.getAnalyzer(in, file); + AbstractAnalyzer fa = new AnalyzerGuru().getAnalyzer(in, file); assertSame(ShAnalyzer.class, fa.getClass()); } @@ -81,7 +81,7 @@ public void testUTF8ByteOrderMark() throws Exception { 'v', 'e', 'r', 's', 'i', 'o', 'n', '=', '"', '1', '.', '0', '"', '?', '>'}; ByteArrayInputStream in = new ByteArrayInputStream(xml); - AbstractAnalyzer fa = AnalyzerGuru.getAnalyzer(in, "/dummy/file"); + AbstractAnalyzer fa = new AnalyzerGuru().getAnalyzer(in, "/dummy/file"); assertSame(XMLAnalyzer.class, fa.getClass()); } @@ -90,7 +90,7 @@ public void testUTF8ByteOrderMarkPlusCopyrightSymbol() throws Exception { byte[] doc = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF, // UTF-8 BOM '/', '/', ' ', (byte) 0xC2, (byte)0xA9}; ByteArrayInputStream in = new ByteArrayInputStream(doc); - AbstractAnalyzer fa = AnalyzerGuru.getAnalyzer(in, "/dummy/file"); + AbstractAnalyzer fa = new AnalyzerGuru().getAnalyzer(in, "/dummy/file"); assertSame("despite BOM as precise match,", PlainAnalyzer.class, fa.getClass()); } @@ -100,9 +100,9 @@ public void testUTF8ByteOrderMarkPlainFile() throws Exception { byte[] bytes = {(byte) 0xEF, (byte) 0xBB, (byte) 0xBF, // UTF-8 BOM 'h', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd'}; - + ByteArrayInputStream in = new ByteArrayInputStream(bytes); - AbstractAnalyzer fa = AnalyzerGuru.getAnalyzer(in, "/dummy/file"); + AbstractAnalyzer fa = new AnalyzerGuru().getAnalyzer(in, "/dummy/file"); assertSame(PlainAnalyzer.class, fa.getClass()); } @@ -111,7 +111,7 @@ public void testUTF16BigByteOrderMarkPlusCopyrightSymbol() throws Exception { byte[] doc = {(byte) 0xFE, (byte) 0xFF, // UTF-16BE BOM 0, '#', 0, ' ', (byte) 0xC2, (byte) 0xA9}; ByteArrayInputStream in = new ByteArrayInputStream(doc); - AbstractAnalyzer fa = AnalyzerGuru.getAnalyzer(in, "/dummy/file"); + AbstractAnalyzer fa = new AnalyzerGuru().getAnalyzer(in, "/dummy/file"); assertSame("despite BOM as precise match,", PlainAnalyzer.class, fa.getClass()); } @@ -121,49 +121,51 @@ public void testUTF16LittleByteOrderMarkPlusCopyrightSymbol() throws Exception { byte[] doc = {(byte) 0xFF, (byte) 0xFE, // UTF-16BE BOM '#', 0, ' ', 0, (byte) 0xA9, (byte) 0xC2}; ByteArrayInputStream in = new ByteArrayInputStream(doc); - AbstractAnalyzer fa = AnalyzerGuru.getAnalyzer(in, "/dummy/file"); + AbstractAnalyzer fa = new AnalyzerGuru().getAnalyzer(in, "/dummy/file"); assertSame("despite BOM as precise match,", PlainAnalyzer.class, fa.getClass()); } @Test public void addExtension() throws Exception { + final AnalyzerGuru analyzerGuru = new AnalyzerGuru(); // should not find analyzer for this unlikely extension - assertNull(AnalyzerGuru.find("file.unlikely_extension")); + assertNull(analyzerGuru.find("file.unlikely_extension")); AnalyzerFactory - faf = AnalyzerGuru.findFactory(ShAnalyzerFactory.class.getName()); + faf = analyzerGuru.findFactory(ShAnalyzerFactory.class.getName()); // should be the same factory as the built-in analyzer for sh scripts - assertSame(AnalyzerGuru.find("myscript.sh"), faf); + assertSame(analyzerGuru.find("myscript.sh"), faf); // add an analyzer for the extension and see that it is picked up - AnalyzerGuru.addExtension("UNLIKELY_EXTENSION", faf); + analyzerGuru.addExtension("UNLIKELY_EXTENSION", faf); assertSame(ShAnalyzerFactory.class, - AnalyzerGuru.find("file.unlikely_extension").getClass()); + analyzerGuru.find("file.unlikely_extension").getClass()); // remove the mapping and verify that it is gone - AnalyzerGuru.addExtension("UNLIKELY_EXTENSION", null); - assertNull(AnalyzerGuru.find("file.unlikely_extension")); + analyzerGuru.addExtension("UNLIKELY_EXTENSION", null); + assertNull(analyzerGuru.find("file.unlikely_extension")); } @Test public void addPrefix() throws Exception { + final AnalyzerGuru analyzerGuru = new AnalyzerGuru(); // should not find analyzer for this unlikely extension - assertNull(AnalyzerGuru.find("unlikely_prefix.foo")); + assertNull(analyzerGuru.find("unlikely_prefix.foo")); AnalyzerFactory - faf = AnalyzerGuru.findFactory(ShAnalyzerFactory.class.getName()); + faf = analyzerGuru.findFactory(ShAnalyzerFactory.class.getName()); // should be the same factory as the built-in analyzer for sh scripts - assertSame(AnalyzerGuru.find("myscript.sh"), faf); + assertSame(analyzerGuru.find("myscript.sh"), faf); // add an analyzer for the prefix and see that it is picked up - AnalyzerGuru.addPrefix("UNLIKELY_PREFIX", faf); + analyzerGuru.addPrefix("UNLIKELY_PREFIX", faf); assertSame(ShAnalyzerFactory.class, - AnalyzerGuru.find("unlikely_prefix.foo").getClass()); + analyzerGuru.find("unlikely_prefix.foo").getClass()); // remove the mapping and verify that it is gone - AnalyzerGuru.addPrefix("UNLIKELY_PREFIX", null); - assertNull(AnalyzerGuru.find("unlikely_prefix.foo")); + analyzerGuru.addPrefix("UNLIKELY_PREFIX", null); + assertNull(analyzerGuru.find("unlikely_prefix.foo")); } @Test @@ -174,7 +176,7 @@ public void testZip() throws IOException { zos.closeEntry(); zos.close(); InputStream in = new ByteArrayInputStream(baos.toByteArray()); - AbstractAnalyzer fa = AnalyzerGuru.getAnalyzer(in, "dummy"); + AbstractAnalyzer fa = new AnalyzerGuru().getAnalyzer(in, "dummy"); assertSame(ZipAnalyzer.class, fa.getClass()); } @@ -186,7 +188,7 @@ public void testJar() throws IOException { jos.closeEntry(); jos.close(); InputStream in = new ByteArrayInputStream(baos.toByteArray()); - AbstractAnalyzer fa = AnalyzerGuru.getAnalyzer(in, "dummy"); + AbstractAnalyzer fa = new AnalyzerGuru().getAnalyzer(in, "dummy"); assertSame(JarAnalyzer.class, fa.getClass()); } @@ -195,21 +197,22 @@ public void testPlainText() throws IOException { ByteArrayInputStream in = new ByteArrayInputStream( "This is a plain text file.".getBytes("US-ASCII")); assertSame(PlainAnalyzer.class, - AnalyzerGuru.getAnalyzer(in, "dummy").getClass()); + new AnalyzerGuru().getAnalyzer(in, "dummy").getClass()); } @Test public void rfe2969() { - AnalyzerFactory faf = AnalyzerGuru.find("foo.hxx"); + AnalyzerFactory faf = new AnalyzerGuru().find("foo.hxx"); assertNotNull(faf); assertSame(CxxAnalyzerFactory.class, faf.getClass()); } @Test public void rfe3401() { - AnalyzerFactory f1 = AnalyzerGuru.find("main.c"); + final AnalyzerGuru analyzerGuru = new AnalyzerGuru(); + AnalyzerFactory f1 = analyzerGuru.find("main.c"); assertNotNull(f1); - AnalyzerFactory f2 = AnalyzerGuru.find("main.cc"); + AnalyzerFactory f2 = analyzerGuru.find("main.cc"); assertNotNull(f2); assertNotSame(f1.getClass(), f2.getClass()); @@ -221,35 +224,36 @@ public void rfe3401() { @Test @SuppressWarnings("rawtypes") public void matchesFullName() { + final AnalyzerGuru analyzerGuru = new AnalyzerGuru(); String s = File.separator; // so test works on Unix and Windows String path = s+"path"+s+"to"+s+"Makefile"; - AnalyzerFactory faf = AnalyzerGuru.find(path); + AnalyzerFactory faf = analyzerGuru.find(path); Class c = faf.getClass(); assertSame(ShAnalyzerFactory.class, faf.getClass()); - faf = AnalyzerGuru.find("GNUMakefile"); + faf = analyzerGuru.find("GNUMakefile"); assertSame(ShAnalyzerFactory.class, faf.getClass()); } - + /** * Test for obtaining a language analyzer's factory class. * This should not fail even if package names change. - * The only assumptions made is that all the language analyzer + * The only assumptions made is that all the language analyzer * and factory names follow the pattern: - * - * language + "Analyzer", and + * + * language + "Analyzer", and * language + "AnalyzerFactory" */ @Test @SuppressWarnings("rawtypes") public void getAnalyzerFactoryClass() { - Class fc_forSh = AnalyzerGuru.getFactoryClass("Sh"); - Class fc_forShAnalyzer = AnalyzerGuru.getFactoryClass("ShAnalyzer"); - Class fc_simpleName = AnalyzerGuru.getFactoryClass("ShAnalyzerFactory"); + Class fc_forSh = new AnalyzerGuru().getFactoryClass("Sh"); + Class fc_forShAnalyzer = new AnalyzerGuru().getFactoryClass("ShAnalyzer"); + Class fc_simpleName = new AnalyzerGuru().getFactoryClass("ShAnalyzerFactory"); assertEquals(ShAnalyzerFactory.class, fc_forSh); assertEquals(ShAnalyzerFactory.class,fc_forShAnalyzer); assertEquals(ShAnalyzerFactory.class,fc_simpleName); - - Class fc = AnalyzerGuru.getFactoryClass("UnknownAnalyzerFactory"); + + Class fc = new AnalyzerGuru().getFactoryClass("UnknownAnalyzerFactory"); assertNull(fc); } @@ -259,7 +263,7 @@ public void shouldNotThrowGettingCsprojOpening() throws IOException { "analysis/a.csproj"); assertNotNull("despite embedded a.csproj,", res); assertSame("despite normal a.csproj,", XMLAnalyzer.class, - AnalyzerGuru.getAnalyzer(res, "dummy").getClass()); + new AnalyzerGuru().getAnalyzer(res, "dummy").getClass()); } @Test @@ -267,7 +271,7 @@ public void shouldMatchPerlHashbang() throws IOException { ByteArrayInputStream in = new ByteArrayInputStream( "#!/usr/bin/perl -w".getBytes("US-ASCII")); assertSame("despite Perl hashbang,", PerlAnalyzer.class, - AnalyzerGuru.getAnalyzer(in, "dummy").getClass()); + new AnalyzerGuru().getAnalyzer(in, "dummy").getClass()); } @Test @@ -275,7 +279,7 @@ public void shouldMatchPerlHashbangSpaced() throws IOException { ByteArrayInputStream in = new ByteArrayInputStream( "\n\t #! /usr/bin/perl -w".getBytes("US-ASCII")); assertSame("despite Perl hashbang,", PerlAnalyzer.class, - AnalyzerGuru.getAnalyzer(in, "dummy").getClass()); + new AnalyzerGuru().getAnalyzer(in, "dummy").getClass()); } @Test @@ -283,7 +287,7 @@ public void shouldMatchEnvPerlHashbang() throws IOException { ByteArrayInputStream in = new ByteArrayInputStream( "#!/usr/bin/env perl -w".getBytes("US-ASCII")); assertSame("despite env hashbang with perl,", PerlAnalyzer.class, - AnalyzerGuru.getAnalyzer(in, "dummy").getClass()); + new AnalyzerGuru().getAnalyzer(in, "dummy").getClass()); } @Test @@ -291,7 +295,7 @@ public void shouldMatchEnvPerlHashbangSpaced() throws IOException { ByteArrayInputStream in = new ByteArrayInputStream( "\n\t #! /usr/bin/env\t perl -w".getBytes("US-ASCII")); assertSame("despite env hashbang with perl,", PerlAnalyzer.class, - AnalyzerGuru.getAnalyzer(in, "dummy").getClass()); + new AnalyzerGuru().getAnalyzer(in, "dummy").getClass()); } @Test @@ -299,7 +303,7 @@ public void shouldNotMatchEnvLFPerlHashbang() throws IOException { ByteArrayInputStream in = new ByteArrayInputStream( "#!/usr/bin/env\nperl".getBytes("US-ASCII")); assertNotSame("despite env hashbang LF,", PerlAnalyzer.class, - AnalyzerGuru.getAnalyzer(in, "dummy").getClass()); + new AnalyzerGuru().getAnalyzer(in, "dummy").getClass()); } @Test @@ -307,7 +311,7 @@ public void shouldMatchELFMagic() throws Exception { byte[] elfmt = {(byte)0x7F, 'E', 'L', 'F', (byte) 2, (byte) 2, (byte) 1, (byte) 0x06}; ByteArrayInputStream in = new ByteArrayInputStream(elfmt); - AbstractAnalyzer fa = AnalyzerGuru.getAnalyzer(in, "/dummy/file"); + AbstractAnalyzer fa = new AnalyzerGuru().getAnalyzer(in, "/dummy/file"); assertSame("despite \\177ELF magic,", ELFAnalyzer.class, fa.getClass()); } @@ -323,7 +327,7 @@ public void shouldMatchJavaClassMagic() throws Exception { byte[] dotclass = {(byte) 0xCA, (byte) 0xFE, (byte) 0xBA, (byte) 0xBE, (byte) 0, (byte) 1, (byte) 0, (byte) 0x34}; ByteArrayInputStream in = new ByteArrayInputStream(dotclass); - AbstractAnalyzer fa = AnalyzerGuru.getAnalyzer(in, "/dummy/file"); + AbstractAnalyzer fa = new AnalyzerGuru().getAnalyzer(in, "/dummy/file"); assertSame("despite 0xCAFEBABE magic,", JavaClassAnalyzer.class, fa.getClass()); } @@ -333,7 +337,7 @@ public void shouldMatchTroffMagic() throws Exception { byte[] mandoc = {' ', '\n', '.', '\"', '\n', '.', 'T', 'H', (byte) 0x20, '\n'}; ByteArrayInputStream in = new ByteArrayInputStream(mandoc); - AbstractAnalyzer fa = AnalyzerGuru.getAnalyzer(in, "/dummy/file"); + AbstractAnalyzer fa = new AnalyzerGuru().getAnalyzer(in, "/dummy/file"); assertSame("despite .TH magic,", TroffAnalyzer.class, fa.getClass()); } @@ -343,7 +347,7 @@ public void shouldMatchMandocMagic() throws Exception { byte[] mandoc = {'\n', ' ', '.', '\"', '\n', '.', 'D', 'd', (byte) 0x20, '\n'}; ByteArrayInputStream in = new ByteArrayInputStream(mandoc); - AbstractAnalyzer fa = AnalyzerGuru.getAnalyzer(in, "/dummy/file"); + AbstractAnalyzer fa = new AnalyzerGuru().getAnalyzer(in, "/dummy/file"); assertSame("despite .Dd magic,", MandocAnalyzer.class, fa.getClass()); } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/LuceneCompatibilityTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/LuceneCompatibilityTest.java deleted file mode 100644 index 3f079d96399..00000000000 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/LuceneCompatibilityTest.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * See LICENSE.txt included in this distribution for the specific - * language governing permissions and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at LICENSE.txt. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -/* - * Copyright (c) 2012, 2018 Oracle and/or its affiliates. All rights reserved. - */ -package org.opengrok.indexer.analysis; - -import java.io.IOException; -import java.io.StringReader; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Iterator; -import junit.framework.Test; -import junit.framework.TestCase; -import junit.framework.TestSuite; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; - -/** - * external tests, need to have test-framework on the path this will do a sanity - * test on analyzers/tokenizers if they follow latest lucene asserts - * - * on compile test cp there needs to be lucene-test-framework, lucene-codecs and - * randomizedtesting-runner on test src path there can be then the whole - * test-framework/src from lucene - * - * @author Lubos Kosco - */ -public class LuceneCompatibilityTest extends TestCase { - - //TODO use reflection to init tests in case LUCENE_TEST_CLASS is present, - // create the object out of it and call it's methods - public LuceneCompatibilityTest() { - super(); - } - private static final String LUCENE_TEST_CLASS = - "org.apache.lucene.analysis.BaseTokenStreamTestCase"; - private static final String LUCENE_TEST_METHOD = "assertTokenStreamContents"; - private static final String LUCENE_DEP = - "com.carrotsearch.randomizedtesting.RandomizedTest"; - - /** - * Create a suite of tests to run. If the lucene test-framework classes are - * not present, skip this test. - * - * @return tests to run - */ - public static Test suite() { - try { - Class.forName(LUCENE_DEP); - Class.forName(LUCENE_TEST_CLASS); - return new TestSuite(LuceneCompatibilityTest.class); - } catch (ClassNotFoundException e) { - return new TestSuite("LuceneCompatibility - empty (no external lucene test framework on classpath)"); - } - } - Analyzer testA; - AnalyzerGuru guru; - Method testM; - Object testC = null; - - /** - * Set up the test environment with repositories and a cache instance. - */ - @Override - @SuppressWarnings("rawtypes") - protected void setUp() throws Exception { - guru = new AnalyzerGuru(); - Class c = Class.forName(LUCENE_TEST_CLASS); - //testC = c.newInstance(); //this is static call - Class[] argTypes = new Class[]{TokenStream.class, String[].class, int[].class, int[].class, String[].class, int[].class, int[].class, Integer.class, boolean.class}; - testM = c.getDeclaredMethod(LUCENE_TEST_METHOD, argTypes); - } - - @Override - protected void tearDown() throws Exception { - } - - @SuppressWarnings("rawtypes") - public void testCompatibility() throws Exception, IOException, IllegalAccessException, IllegalArgumentException { - for (Iterator it = guru.getAnalyzerFactories().iterator(); it.hasNext();) { - AnalyzerFactory fa = (AnalyzerFactory) it.next(); - String input = "Hello world"; - String[] output = new String[]{"Hello", "world"}; - testA = fa.getAnalyzer(); - String name = testA.getClass().getName(); - //below analyzers have no refs - - // !!!!!!!!!!!!!!!!!!!! - // below will fail for some analyzers because of the way how we - // deal with data - we don't use the reader, but cache the whole - // file instead inside "content" buffer (which is reused for xref) - // !!!!!!!!!!!!!!!!!!!! - try { - if (!name.endsWith("FileAnalyzer") && !name.endsWith("BZip2Analyzer") && !name.endsWith("GZIPAnalyzer") - && !name.endsWith("XMLAnalyzer") && !name.endsWith("TroffAnalyzer") && !name.endsWith("ELFAnalyzer") - && !name.endsWith("JavaClassAnalyzer") && !name.endsWith("JarAnalyzer") && !name.endsWith("ZipAnalyzer") - //TODO below php and fortran analyzers have some problems with dummy input and asserts fail, - // analyzers should properly set the tokens in case of wrongly formulated input - && !name.endsWith("TarAnalyzer") && !name.endsWith("PhpAnalyzer") && !name.endsWith("FortranAnalyzer")) { - - System.out.println("Testing refs with " + name); - //BaseTokenStreamTestCase.assertTokenStreamContents(testA.tokenStream("refs", new StringReader(input)), output, null, null, null, null, null, input.length()); - testM.invoke(testC, testA.tokenStream("refs", new StringReader(input)), output, null, null, null, null, null, input.length(), true); - } - output = new String[]{"hello", "world"}; - //below analyzers have no full, they just wrap data inside them - if (!name.endsWith("FileAnalyzer") && !name.endsWith("BZip2Analyzer") && !name.endsWith("GZIPAnalyzer")) { - System.out.println("Testing full with " + name); - //BaseTokenStreamTestCase.assertTokenStreamContents(testA.tokenStream("full", new StringReader(input)), output, null, null, null, null, null, input.length()); - testM.invoke(testC, testA.tokenStream("full", new StringReader(input)), output, null, null, null, null, null, input.length(), true); - } - } catch (InvocationTargetException x) { - Throwable cause = x.getCause(); - System.err.println(name + " failed: " + cause.getMessage() + " from " + LUCENE_TEST_CLASS + ":" + LUCENE_TEST_METHOD); - throw (new Exception(cause)); - } - } - } -} diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/archive/ZipAnalyzerFactoryTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/archive/ZipAnalyzerFactoryTest.java index 12d43697e17..88e5a69d048 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/archive/ZipAnalyzerFactoryTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/archive/ZipAnalyzerFactoryTest.java @@ -23,10 +23,11 @@ package org.opengrok.indexer.analysis.archive; -import java.io.IOException; -import java.io.InputStream; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertSame; + +import java.io.IOException; +import java.io.InputStream; import org.junit.Test; import org.opengrok.indexer.analysis.AnalyzerFactory; import org.opengrok.indexer.analysis.AnalyzerGuru; @@ -38,18 +39,19 @@ public class ZipAnalyzerFactoryTest { /** * Tests a ZIP file. + * * @throws IOException I/O exception */ @Test public void testZipWrtAnalyzerGuru() throws IOException { InputStream res = getClass().getClassLoader().getResourceAsStream( - "analysis/archive/zip.bin"); + "analysis/archive/zip.bin"); assertNotNull("zip.bin should be available,", res); // assert that it is matched - AnalyzerFactory fac = AnalyzerGuru.find(res); + AnalyzerFactory fac = new AnalyzerGuru().find(res); assertNotNull("zip.bin should have factory", fac); assertSame("should be ZipAnalyzerFactory", fac.getClass(), - ZipAnalyzerFactory.class); + ZipAnalyzerFactory.class); } } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerFactoryTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerFactoryTest.java index 661cc9bd569..c69bac95460 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerFactoryTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JarAnalyzerFactoryTest.java @@ -29,6 +29,7 @@ import static org.junit.Assert.assertSame; import org.junit.Test; import org.opengrok.indexer.analysis.AnalyzerFactory; +import org.opengrok.indexer.analysis.AnalyzerFramework; import org.opengrok.indexer.analysis.AnalyzerGuru; /** @@ -47,7 +48,7 @@ public void testJarWrtAnalyzerGuru() throws IOException { assertNotNull("javajar.bin should be available,", res); // assert that it is matched - AnalyzerFactory fac = AnalyzerGuru.find(res); + AnalyzerFactory fac = new AnalyzerGuru().find(res); assertNotNull("javajar.bin should have factory", fac); assertSame("should be JarAnalyzerFactory", fac.getClass(), JarAnalyzerFactory.class); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JavaClassAnalyzerFactoryTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JavaClassAnalyzerFactoryTest.java index a6123eec18d..01a41a28a2a 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JavaClassAnalyzerFactoryTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/analysis/executables/JavaClassAnalyzerFactoryTest.java @@ -30,6 +30,7 @@ import static org.junit.Assert.assertSame; import org.junit.Test; import org.opengrok.indexer.analysis.AnalyzerFactory; +import org.opengrok.indexer.analysis.AnalyzerFramework; import org.opengrok.indexer.analysis.AnalyzerGuru; /** @@ -48,7 +49,7 @@ public void testJavaClassWrtAnalyzerGuru() throws IOException { assertNotNull("despite inclusion locally,", res); // assert that it is matched - AnalyzerFactory fac = AnalyzerGuru.find(res); + AnalyzerFactory fac = new AnalyzerGuru().find(res); assertNotNull("javaclass.bin should have factory", fac); assertSame("should be JavaClassAnalyzerFactory", fac.getClass(), JavaClassAnalyzerFactory.class); @@ -64,7 +65,7 @@ public void testDylibCafebabeWrtAnalyzerGuru() throws IOException { "analysis/executables/fat.dylib"); assertNotNull("despite inclusion locally,", res); - AnalyzerFactory fac = AnalyzerGuru.find(res); + AnalyzerFactory fac = new AnalyzerGuru().find(res); if (fac != null) { assertNotSame("should not be JavaClassAnalyzerFactory", fac.getClass(), JavaClassAnalyzerFactory.class); diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java index d62fa5a91da..d151c94ae74 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/index/IndexerTest.java @@ -53,6 +53,7 @@ import org.junit.Rule; import org.junit.Test; import org.opengrok.indexer.analysis.AnalyzerFactory; +import org.opengrok.indexer.analysis.AnalyzerFramework; import org.opengrok.indexer.analysis.AnalyzerGuru; import org.opengrok.indexer.condition.ConditionalRun; import org.opengrok.indexer.condition.ConditionalRunRule; @@ -358,8 +359,9 @@ public void testRemoveFileOnFileChange() throws Exception { public void testXref() throws IOException { List files = new ArrayList<>(); FileUtilities.getAllFiles(new File(repository.getSourceRoot()), files, false); + final AnalyzerGuru guru = new AnalyzerGuru(); for (File f : files) { - AnalyzerFactory factory = AnalyzerGuru.find(f.getAbsolutePath()); + AnalyzerFactory factory = guru.find(f.getAbsolutePath()); if (factory == null) { continue; } diff --git a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/PageConfigRequestedProjectsTest.java b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/PageConfigRequestedProjectsTest.java index a698f371fea..14ea32ea7e5 100644 --- a/opengrok-indexer/src/test/java/org/opengrok/indexer/web/PageConfigRequestedProjectsTest.java +++ b/opengrok-indexer/src/test/java/org/opengrok/indexer/web/PageConfigRequestedProjectsTest.java @@ -12,6 +12,7 @@ import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; +import org.opengrok.indexer.authorization.AuthControlFlag; import org.opengrok.indexer.authorization.AuthorizationStack; import org.opengrok.indexer.configuration.Group; import org.opengrok.indexer.configuration.Project; diff --git a/opengrok-indexer/src/test/resources/analysis/plugins/testplugins.jar b/opengrok-indexer/src/test/resources/analysis/plugins/testplugins.jar new file mode 100644 index 00000000000..5d2ca26c017 Binary files /dev/null and b/opengrok-indexer/src/test/resources/analysis/plugins/testplugins.jar differ diff --git a/opengrok-web/src/main/java/org/opengrok/web/WebappListener.java b/opengrok-web/src/main/java/org/opengrok/web/WebappListener.java index 39714e532b6..ea40b05fc27 100644 --- a/opengrok-web/src/main/java/org/opengrok/web/WebappListener.java +++ b/opengrok-web/src/main/java/org/opengrok/web/WebappListener.java @@ -23,8 +23,19 @@ */ package org.opengrok.web; +import static org.opengrok.indexer.util.StatisticsUtils.loadStatistics; +import static org.opengrok.indexer.util.StatisticsUtils.saveStatistics; + +import java.io.File; +import java.io.IOException; +import java.util.logging.Level; +import java.util.logging.Logger; +import javax.servlet.ServletContext; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; +import javax.servlet.ServletRequestEvent; +import javax.servlet.ServletRequestListener; import org.opengrok.indexer.Info; -import org.opengrok.indexer.analysis.AnalyzerGuru; import org.opengrok.indexer.authorization.AuthorizationFramework; import org.opengrok.indexer.configuration.RuntimeEnvironment; import org.opengrok.indexer.logger.LoggerFactory; @@ -32,19 +43,6 @@ import org.opengrok.indexer.web.SearchHelper; import org.opengrok.web.api.v1.suggester.provider.service.SuggesterServiceFactory; -import javax.servlet.ServletContext; -import javax.servlet.ServletContextEvent; -import javax.servlet.ServletContextListener; -import javax.servlet.ServletRequestEvent; -import javax.servlet.ServletRequestListener; -import java.io.File; -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static org.opengrok.indexer.util.StatisticsUtils.loadStatistics; -import static org.opengrok.indexer.util.StatisticsUtils.saveStatistics; - /** * Initialize webapp context * @@ -65,7 +63,7 @@ public void contextInitialized(final ServletContextEvent servletContextEvent) { LOGGER.log(Level.INFO, "Starting webapp with version {0} ({1})", new Object[]{Info.getVersion(), Info.getRevision()}); - + String config = context.getInitParameter("CONFIGURATION"); if (config == null) { LOGGER.severe("CONFIGURATION section missing in web.xml"); @@ -85,6 +83,10 @@ public void contextInitialized(final ServletContextEvent servletContextEvent) { env.setAuthorizationFramework(new AuthorizationFramework(env.getPluginDirectory(), env.getPluginStack())); env.getAuthorizationFramework().reload(); + // set the new plugin directory and reload the analyzer framework + env.getAnalyzerGuru().getPluginFramework().setPluginDirectory(env.getPluginDirectory()); + env.getAnalyzerGuru().getPluginFramework().reload(); + try { loadStatistics(); } catch (IOException ex) { @@ -136,7 +138,7 @@ public void requestDestroyed(ServletRequestEvent e) { sh.destroy(); } - AnalyzerGuru.returnAnalyzers(); + RuntimeEnvironment.getInstance().getAnalyzerGuru().returnAnalyzers(); } /** diff --git a/opengrok-web/src/main/webapp/list.jsp b/opengrok-web/src/main/webapp/list.jsp index ac1ab9515a9..751eeba9db1 100644 --- a/opengrok-web/src/main/webapp/list.jsp +++ b/opengrok-web/src/main/webapp/list.jsp @@ -33,15 +33,15 @@ java.net.URLEncoder, java.nio.charset.StandardCharsets, java.util.List, java.util.Locale, -java.util.logging.Level, java.util.Set, +java.util.logging.Level, java.util.logging.Logger, -org.opengrok.indexer.analysis.AnalyzerGuru, -org.opengrok.indexer.analysis.Ctags, -org.opengrok.indexer.analysis.Definitions, org.opengrok.indexer.analysis.AbstractAnalyzer, org.opengrok.indexer.analysis.AbstractAnalyzer.Genre, org.opengrok.indexer.analysis.AnalyzerFactory, +org.opengrok.indexer.analysis.AnalyzerGuru, +org.opengrok.indexer.analysis.Ctags, +org.opengrok.indexer.analysis.Definitions, org.opengrok.indexer.history.Annotation, org.opengrok.indexer.index.IndexDatabase, org.opengrok.indexer.logger.LoggerFactory, @@ -49,10 +49,10 @@ org.opengrok.indexer.search.DirectoryEntry, org.opengrok.indexer.search.DirectoryExtraReader, org.opengrok.indexer.search.FileExtra, org.opengrok.indexer.util.FileExtraZipper, -org.opengrok.indexer.util.ObjectPool, org.opengrok.indexer.util.IOUtils, -org.opengrok.web.DirectoryListing, -org.opengrok.indexer.web.SearchHelper" +org.opengrok.indexer.util.ObjectPool, +org.opengrok.indexer.web.SearchHelper, +org.opengrok.web.DirectoryListing" %> <% { @@ -186,11 +186,11 @@ document.pageReady.push(function() { pageReadyList();}); BufferedInputStream bin = new BufferedInputStream(new FileInputStream(resourceFile)); try { - AnalyzerFactory a = AnalyzerGuru.find(basename); - AbstractAnalyzer.Genre g = AnalyzerGuru.getGenre(a); + AnalyzerFactory a = cfg.getEnv().getAnalyzerGuru().find(basename); + AbstractAnalyzer.Genre g = cfg.getEnv().getAnalyzerGuru().getGenre(a); if (g == null) { - a = AnalyzerGuru.find(bin); - g = AnalyzerGuru.getGenre(a); + a = cfg.getEnv().getAnalyzerGuru().find(bin); + g = cfg.getEnv().getAnalyzerGuru().getGenre(a); } if (g == AbstractAnalyzer.Genre.IMAGE) { %> @@ -247,8 +247,8 @@ Click download <%= basename %><% } } else { // requesting a previous revision or needed to generate xref on the fly (economy mode). - AnalyzerFactory a = AnalyzerGuru.find(basename); - Genre g = AnalyzerGuru.getGenre(a); + AnalyzerFactory a = cfg.getEnv().getAnalyzerGuru().find(basename); + AbstractAnalyzer.Genre g = cfg.getEnv().getAnalyzerGuru().getGenre(a); String error = null; if (g == Genre.PLAIN || g == Genre.HTML || g == null) { InputStream in = null; @@ -278,8 +278,8 @@ Click download <%= basename %><% if (in != null) { try { if (g == null) { - a = AnalyzerGuru.find(in, basename); - g = AnalyzerGuru.getGenre(a); + a = cfg.getEnv().getAnalyzerGuru().find(in, basename); + g = cfg.getEnv().getAnalyzerGuru().getGenre(a); } if (g == AbstractAnalyzer.Genre.DATA || g == AbstractAnalyzer.Genre.XREFABLE || g == null) { %>