// // QueryDriver.cs // // Copyright (C) 2004 Novell, Inc. // // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), // to deal in the Software without restriction, including without limitation // the rights to use, copy, modify, merge, publish, distribute, sublicense, // and/or sell copies of the Software, and to permit persons to whom the // Software is furnished to do so, subject to the following conditions: // // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. // using System; using System.IO; using System.Collections; using System.Reflection; using System.Text; using System.Threading; using Beagle.Util; namespace Beagle.Daemon { public class QueryDriver { // Contains list of queryables explicitly asked by --allow-backend or --backend name // --allow-backend/--backend name : dont read config information and only start backend 'name' static ArrayList excl_allowed_queryables = new ArrayList (); // Contains list of denied queryables from config/arguments (every queryable is enabled by default) // Unless overruled by --allow-backend/--backend name, deny backend only if names appears here. static ArrayList denied_queryables = new ArrayList (); static bool to_read_conf = true; // read backends from conf if true static bool done_reading_conf = false; static private void ReadBackendsFromConf () { if (! to_read_conf || done_reading_conf) return; // set flag here to stop Allow() from calling ReadBackendsFromConf() again done_reading_conf = true; // To allow static indexes, "static" should be in allowed_queryables if (Conf.Daemon.AllowStaticBackend) Allow ("static"); if (Conf.Daemon.DeniedBackends == null) return; foreach (string name in Conf.Daemon.DeniedBackends) denied_queryables.Add (name.ToLower ()); } static public void OnlyAllow (string name) { excl_allowed_queryables.Add (name.ToLower ()); to_read_conf = false; } static public void Allow (string name) { if (! done_reading_conf && to_read_conf) ReadBackendsFromConf (); denied_queryables.Remove (name.ToLower ()); } static public void Deny (string name) { if (! done_reading_conf && to_read_conf) ReadBackendsFromConf (); name = name.ToLower (); if (!denied_queryables.Contains (name)) denied_queryables.Add (name); } static private bool UseQueryable (string name) { name = name.ToLower (); if (excl_allowed_queryables.Contains (name)) return true; if (excl_allowed_queryables.Count != 0) return false; if (denied_queryables.Contains (name)) return false; return true; } ////////////////////////////////////////////////////////////////////////////////////// // Paths to static queryables static ArrayList static_queryables = new ArrayList (); static public void AddStaticQueryable (string path) { if (! static_queryables.Contains (path)) static_queryables.Add (path); } ////////////////////////////////////////////////////////////////////////////////////// // Delay before starting the indexing process static int indexing_delay = 60; // Default to 60 seconds public static int IndexingDelay { set { indexing_delay = value; } } ////////////////////////////////////////////////////////////////////////////////////// // Use introspection to find all classes that implement IQueryable, the construct // associated Queryables objects. static ArrayList queryables = new ArrayList (); static Hashtable iqueryable_to_queryable = new Hashtable (); static bool ThisApiSoVeryIsBroken (Type m, object criteria) { return m == (Type) criteria; } static bool TypeImplementsInterface (Type t, Type iface) { Type[] impls = t.FindInterfaces (new TypeFilter (ThisApiSoVeryIsBroken), iface); return impls.Length > 0; } // For every type in the assembly that // (1) implements IQueryable // (2) Has a QueryableFlavor attribute attached // assemble a Queryable object and stick it into our list of queryables. static void ScanAssembly (Assembly assembly) { int count = 0; foreach (Type type in ReflectionFu.ScanAssemblyForInterface (assembly, typeof (IQueryable))) { bool type_accepted = false; foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor))) { if (! UseQueryable (flavor.Name)) continue; if (flavor.RequireInotify && ! Inotify.Enabled) { Logger.Log.Warn ("Can't start backend '{0}' without inotify", flavor.Name); continue; } if (flavor.RequireExtendedAttributes && ! ExtendedAttribute.Supported) { Logger.Log.Warn ("Can't start backend '{0}' without extended attributes", flavor.Name); continue; } IQueryable iq = null; try { iq = Activator.CreateInstance (type) as IQueryable; } catch (Exception e) { Logger.Log.Error ("Caught exception while instantiating {0} backend", flavor.Name); Logger.Log.Error (e); } if (iq != null) { Queryable q = new Queryable (flavor, iq); queryables.Add (q); iqueryable_to_queryable [iq] = q; ++count; type_accepted = true; break; } } if (! type_accepted) continue; object[] attributes = type.GetCustomAttributes (false); foreach (object attribute in attributes) { PropertyKeywordMapping mapping = attribute as PropertyKeywordMapping; if (mapping == null) continue; //Logger.Log.Debug (mapping.Keyword + " => " // + mapping.PropertyName + // + " is-keyword=" + mapping.IsKeyword + " (" // + mapping.Description + ") " // + "(" + type.FullName + ")"); PropertyKeywordFu.RegisterMapping (mapping); } } Logger.Log.Debug ("Found {0} backends in {1}", count, assembly.Location); } //////////////////////////////////////////////////////// public static void ReadKeywordMappings () { Logger.Log.Debug ("Reading mapping from filters"); ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_FILTER_PATH", PathFinder.FilterDir); foreach (Assembly assembly in assemblies) { foreach (Type type in assembly.GetTypes ()) if (type.FullName.StartsWith ("Beagle.Filters")) { if (type.IsNestedPrivate || type.IsNestedPublic || type.IsNestedAssembly || type.IsNestedFamily) continue; object[] attributes = type.GetCustomAttributes (false); foreach (object attribute in attributes) { PropertyKeywordMapping mapping = attribute as PropertyKeywordMapping; if (mapping == null) continue; //Logger.Log.Debug (mapping.Keyword + " => " // + mapping.PropertyName // + " is-keyword=" + mapping.IsKeyword + " (" // + mapping.Description + ") " // + "(" + type.FullName + ")"); PropertyKeywordFu.RegisterMapping (mapping); } } } } //////////////////////////////////////////////////////// // Scans PathFinder.SystemIndexesDir after available // system-wide indexes. static void LoadSystemIndexes () { if (!Directory.Exists (PathFinder.SystemIndexesDir)) return; Logger.Log.Info ("Loading system static indexes."); int count = 0; foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) { if (! UseQueryable (index_dir.Name)) continue; if (LoadStaticQueryable (index_dir, QueryDomain.System)) count++; } Logger.Log.Info ("Found {0} system-wide indexes.", count); } // Scans configuration for user-specified index paths // to load StaticQueryables from. static void LoadStaticQueryables () { int count = 0; if (UseQueryable ("static")) { Logger.Log.Info ("Loading user-configured static indexes."); foreach (string path in Conf.Daemon.StaticQueryables) static_queryables.Add (path); } foreach (string path in static_queryables) { DirectoryInfo index_dir = new DirectoryInfo (StringFu.SanitizePath (path)); if (!index_dir.Exists) continue; // FIXME: QueryDomain might be other than local if (LoadStaticQueryable (index_dir, QueryDomain.Local)) count++; } Logger.Log.Info ("Found {0} user-configured static indexes..", count); } // Instantiates and loads a StaticQueryable from an index directory static private bool LoadStaticQueryable (DirectoryInfo index_dir, QueryDomain query_domain) { StaticQueryable static_queryable = null; if (!index_dir.Exists) return false; try { static_queryable = new StaticQueryable (index_dir.Name, index_dir.FullName, true); } catch (InvalidOperationException) { Logger.Log.Warn ("Unable to create read-only index (likely due to index version mismatch): {0}", index_dir.FullName); return false; } catch (Exception e) { Logger.Log.Error ("Caught exception while instantiating static queryable: {0}", index_dir.Name); Logger.Log.Error (e); return false; } if (static_queryable != null) { QueryableFlavor flavor = new QueryableFlavor (); flavor.Name = index_dir.Name; flavor.Domain = query_domain; Queryable queryable = new Queryable (flavor, static_queryable); queryables.Add (queryable); iqueryable_to_queryable [static_queryable] = queryable; return true; } return false; } //////////////////////////////////////////////////////// static public void Start () { ReadBackendsFromConf (); ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir); // Only add the executing assembly if we haven't already loaded it. if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1) assemblies.Add (Assembly.GetExecutingAssembly ()); foreach (Assembly assembly in assemblies) { ScanAssembly (assembly); // This allows backends to define their // own executors. Server.ScanAssemblyForExecutors (assembly); } ReadKeywordMappings (); LoadSystemIndexes (); LoadStaticQueryables (); if (indexing_delay <= 0 || Environment.GetEnvironmentVariable ("BEAGLE_EXERCISE_THE_DOG") != null) StartQueryables (); else { Logger.Log.Debug ("Waiting {0} seconds before starting queryables", indexing_delay); GLib.Timeout.Add ((uint) indexing_delay * 1000, new GLib.TimeoutHandler (StartQueryables)); } } static private bool StartQueryables () { Logger.Log.Debug ("Starting queryables"); foreach (Queryable q in queryables) { Logger.Log.Info ("Starting backend: '{0}'", q.Name); q.Start (); } return false; } static public string ListBackends () { ArrayList assemblies = ReflectionFu.ScanEnvironmentForAssemblies ("BEAGLE_BACKEND_PATH", PathFinder.BackendDir); // Only add the executing assembly if we haven't already loaded it. if (assemblies.IndexOf (Assembly.GetExecutingAssembly ()) == -1) assemblies.Add (Assembly.GetExecutingAssembly ()); string ret = "User:\n"; foreach (Assembly assembly in assemblies) { foreach (Type type in ReflectionFu.ScanAssemblyForInterface (assembly, typeof (IQueryable))) { foreach (QueryableFlavor flavor in ReflectionFu.ScanTypeForAttribute (type, typeof (QueryableFlavor))) ret += String.Format (" - {0}\n", flavor.Name); } } if (!Directory.Exists (PathFinder.SystemIndexesDir)) return ret; ret += "System:\n"; foreach (DirectoryInfo index_dir in new DirectoryInfo (PathFinder.SystemIndexesDir).GetDirectories ()) { ret += String.Format (" - {0}\n", index_dir.Name); } return ret; } static public Queryable GetQueryable (string name) { foreach (Queryable q in queryables) { if (q.Name == name) return q; } return null; } static public Queryable GetQueryable (IQueryable iqueryable) { return (Queryable) iqueryable_to_queryable [iqueryable]; } //////////////////////////////////////////////////////// public delegate void ChangedHandler (Queryable queryable, IQueryableChangeData changeData); static public event ChangedHandler ChangedEvent; // A method to fire the ChangedEvent event. static public void QueryableChanged (IQueryable iqueryable, IQueryableChangeData change_data) { if (ChangedEvent != null) { Queryable queryable = iqueryable_to_queryable [iqueryable] as Queryable; ChangedEvent (queryable, change_data); } } //////////////////////////////////////////////////////// private class QueryClosure : IQueryWorker { Queryable queryable; Query query; IQueryResult result; IQueryableChangeData change_data; public QueryClosure (Queryable queryable, Query query, QueryResult result, IQueryableChangeData change_data) { this.queryable = queryable; this.query = query; this.result = result; this.change_data = change_data; } public void DoWork () { queryable.DoQuery (query, result, change_data); } } static public void DoOneQuery (Queryable queryable, Query query, QueryResult result, IQueryableChangeData change_data) { if (queryable.AcceptQuery (query)) { QueryClosure qc = new QueryClosure (queryable, query, result, change_data); result.AttachWorker (qc); } } static void AddSearchTermInfo (QueryPart part, SearchTermResponse response, StringBuilder sb) { if (part.Logic == QueryPartLogic.Prohibited) return; if (part is QueryPart_Or) { ICollection sub_parts; sub_parts = ((QueryPart_Or) part).SubParts; foreach (QueryPart qp in sub_parts) AddSearchTermInfo (qp, response, sb); return; } if (! (part is QueryPart_Text)) return; QueryPart_Text tp; tp = (QueryPart_Text) part; string [] split; split = tp.Text.Split (' '); // First, remove stop words for (int i = 0; i < split.Length; ++i) if (LuceneCommon.IsStopWord (split [i])) split [i] = null; // Assemble the phrase minus stop words sb.Length = 0; for (int i = 0; i < split.Length; ++i) { if (split [i] == null) continue; if (sb.Length > 0) sb.Append (' '); sb.Append (split [i]); } response.ExactText.Add (sb.ToString ()); // Now assemble a stemmed version sb.Length = 0; // clear the previous value for (int i = 0; i < split.Length; ++i) { if (split [i] == null) continue; if (sb.Length > 0) sb.Append (' '); sb.Append (LuceneCommon.Stem (split [i])); } response.StemmedText.Add (sb.ToString ()); } //////////////////////////////////////////////////////// static private void DehumanizeQuery (Query query) { // We need to remap any QueryPart_Human parts into // lower-level part types. First, we find any // QueryPart_Human parts and explode them into // lower-level types. ArrayList new_parts = null; foreach (QueryPart abstract_part in query.Parts) { if (abstract_part is QueryPart_Human) { QueryPart_Human human = abstract_part as QueryPart_Human; if (new_parts == null) new_parts = new ArrayList (); foreach (QueryPart sub_part in QueryStringParser.Parse (human.QueryString)) new_parts.Add (sub_part); } } // If we found any QueryPart_Human parts, copy the // non-Human parts over and then replace the parts in // the query. if (new_parts != null) { foreach (QueryPart abstract_part in query.Parts) { if (! (abstract_part is QueryPart_Human)) new_parts.Add (abstract_part); } query.ClearParts (); foreach (QueryPart part in new_parts) query.AddPart (part); } } static private SearchTermResponse AssembleSearchTermResponse (Query query) { StringBuilder sb = new StringBuilder (); SearchTermResponse search_term_response; search_term_response = new SearchTermResponse (); foreach (QueryPart part in query.Parts) AddSearchTermInfo (part, search_term_response, sb); return search_term_response; } static private void QueryEachQueryable (Query query, QueryResult result) { // The extra pair of calls to WorkerStart/WorkerFinished ensures: // (1) that the QueryResult will fire the StartedEvent // and FinishedEvent, even if no queryable accepts the // query. // (2) that the FinishedEvent will only get called when all of the // backends have had time to finish. object dummy_worker = new object (); if (! result.WorkerStart (dummy_worker)) return; foreach (Queryable queryable in queryables) DoOneQuery (queryable, query, result, null); result.WorkerFinished (dummy_worker); } static public void DoQueryLocal (Query query, QueryResult result) { DehumanizeQuery (query); SearchTermResponse search_term_response; search_term_response = AssembleSearchTermResponse (query); query.ProcessSearchTermResponse (search_term_response); QueryEachQueryable (query, result); } static public void DoQuery (Query query, QueryResult result, RequestMessageExecutor.AsyncResponse send_response) { DehumanizeQuery (query); SearchTermResponse search_term_response; search_term_response = AssembleSearchTermResponse (query); send_response (search_term_response); QueryEachQueryable (query, result); } //////////////////////////////////////////////////////// static public string GetIndexInformation () { StringBuilder builder = new StringBuilder ('\n'); foreach (Queryable q in queryables) { QueryableStatus status = q.GetQueryableStatus (); builder.Append ("Name: ").Append (status.Name).Append ('\n'); builder.Append ("Count: ").Append (status.ItemCount).Append ('\n'); builder.Append ("Indexing: ").Append (status.IsIndexing).Append ('\n'); builder.Append ('\n'); } return builder.ToString (); } //////////////////////////////////////////////////////// static public bool IsIndexing { get { foreach (Queryable q in queryables) { QueryableStatus status = q.GetQueryableStatus (); if (status.IsIndexing) return true; } return false; } } } }