using System; using System.Collections.Generic; using System.ComponentModel; using System.IO; using System.Linq; using InABox.Clients; using Newtonsoft.Json.Linq; namespace InABox.Core { public abstract class BaseImporter : IImporter, IDisposable where T : Entity, IRemotable, IPersistent, new() { private readonly List _log = new List(); public BaseImporter() { Properties = CoreUtils.PropertyList(typeof(T), x => true, true).Keys.ToArray(); Fields = new string[] { }; Mappings = new List(); HasHeader = true; HeaderRow = 1; } public void Dispose() { Close(); Properties = null; Fields = null; Mappings = null; } public string[] Properties { get; private set; } public bool HasHeader { get; set; } public int HeaderRow { get; set; } public abstract bool ReadHeader(); public string[] Fields { get; protected set; } public List Mappings { get; private set; } public IEnumerable Log => _log; public event ImportNotificationEvent OnNotify; public void InitialiseFields(string[] fields) { Fields = fields; } public abstract bool Open(Stream stream); public abstract void Close(); public abstract bool MoveNext(); public abstract Dictionary ReadLine(); public event ImportPreProcessEvent BeforeProcess; public event ImportPostProcessEvent AfterProcess; public event ImportLoadEvent OnLoad; public event ImportSaveEvent OnSave; public event ImportLogEvent OnLog; public event ImportLogEvent OnError; private void ImportValues(T item, ImportMapping mapping, string value) { var p2 = DatabaseSchema.Property(typeof(T), mapping.Property); //var prop = CoreUtils.GetProperty(typeof(T), property); var type = p2.PropertyType; item.OriginalValueList[mapping.Property] = CoreUtils.GetPropertyValue(item, mapping.Property); if (type == typeof(string)) { if (p2.Editor is UniqueCodeEditor || p2.Editor is CodeEditor) CoreUtils.SetPropertyValue(item, mapping.Property, value?.ToUpper()); else CoreUtils.SetPropertyValue(item, mapping.Property, value); } else { if (string.IsNullOrWhiteSpace(value)) { var def = type.GetDefault(); CoreUtils.SetPropertyValue(item, mapping.Property, def); } else { var converter = TypeDescriptor.GetConverter(type); var converted = converter.ConvertFrom(value); CoreUtils.SetPropertyValue(item, mapping.Property, converted); } } } private void DoLookup(T item, ImportMapping mapping, List lookups) { var parentprop = string.Join(".", mapping.Property.Split('.').Reverse().Skip(1).Reverse()); var parent = CoreUtils.GetProperty(typeof(T), parentprop); var childprop = mapping.Property.Split('.').Last(); var bt = parent.PropertyType.BaseType; if (bt != null) { var lookuptype = bt.GetGenericArguments().FirstOrDefault(); var lookup = lookups.FirstOrDefault(x => x.Type.Equals(lookuptype)); IEnumerable lookuprows = lookup.Results.Rows; var lookupvalue = CoreUtils.GetPropertyValue(item, mapping.Property) as string; if (!string.IsNullOrWhiteSpace(lookupvalue)) { lookuprows = lookuprows.Where(r => r.Get(childprop).Equals(lookupvalue)); var lookupid = lookuprows.Any() ? lookuprows.First().Get("ID") : Guid.Empty; if (lookupid == Guid.Empty) { if (mapping.Lookup == ImportLookupType.Restrict) { throw new Exception(string.Format("Lookup Value [{0}] not found", lookupvalue)); } else if (mapping.Lookup == ImportLookupType.Create) { var newlookup = Activator.CreateInstance(lookuptype); CoreUtils.SetPropertyValue(newlookup, childprop, lookupvalue); ClientFactory.CreateClient(lookuptype).Save(newlookup, "Created by Import"); lookupid = (Guid?)CoreUtils.GetPropertyValue(newlookup, "ID") ?? Guid.Empty; var newrow = lookup.Results.NewRow(); lookup.Results.FillRow(newrow, newlookup); lookup.Results.Rows.Add(newrow); CoreUtils.SetPropertyValue(item, lookup.ID, lookupid); var prefix = String.Join(".", lookup.ID.Split('.').Reverse().Skip(1).Reverse()); foreach (var field in lookup.Fields) CoreUtils.SetPropertyValue(item, String.Join(".", new String[] { prefix, field }), newrow[field]); } } else { CoreUtils.SetPropertyValue(item, lookup.ID, lookupid); var prefix = String.Join(".", lookup.ID.Split('.').Reverse().Skip(1).Reverse()); foreach (var field in lookup.Fields) CoreUtils.SetPropertyValue(item, String.Join(".", new String[] { prefix, field }), lookuprows.First()[field]); } } } } public int Import() { _log.Clear(); _log.Add(string.Format("Import Log {0:dd/MM/yyyy hh:mm:ss}", DateTime.Now)); _log.Add("=============================="); var iResult = 0; ImportLookup? keylookup = null; var lookups = new List(); Notify("Preparing Import Mappings..."); foreach (var mapping in Mappings) { if (mapping.Key) { if (keylookup == null) { keylookup = new ImportLookup(typeof(T), "ID", OnLoad); var others = LookupFactory.DefineColumns(typeof(T)); foreach (var other in others.ColumnNames()) keylookup.Fields.Add(other); lookups.Add(keylookup); } if (!keylookup.Fields.Contains(mapping.Property)) keylookup.Fields.Add(mapping.Property); } if (mapping.Lookup != ImportLookupType.None) { var parentprop = string.Join(".", mapping.Property.Split('.').Reverse().Skip(1).Reverse()); var parent = CoreUtils.GetProperty(typeof(T), parentprop); var childprop = mapping.Property.Split('.').Last(); var bt = parent.PropertyType.BaseType; if (bt != null) { var lookuptype = bt.GetGenericArguments().FirstOrDefault(); var lookup = lookups.FirstOrDefault(x => x.Type == lookuptype); if (lookup == null) { lookup = new ImportLookup(lookuptype, parentprop + ".ID", null); var others = LookupFactory.DefineColumns(lookuptype); foreach (var other in others.ColumnNames()) lookup.Fields.Add(other); lookups.Add(lookup); } if (!lookup.Fields.Contains(childprop)) lookup.Fields.Add(childprop); } } } if (keylookup == null) { WriteLog(0, "WARNING: No Key Fields Found - All Items will be treated as new"); //return iResult; } else { Notify(string.Format("Loading {0} Keys [{1}]", keylookup.Type.Name, string.Join("]+[", keylookup.Fields))); keylookup.Refresh(); } foreach (var lookup in lookups) if (lookup.Results == null) { Notify(string.Format("Loading {0} Lookups [{1}]", lookup.Type.Name, string.Join("]+[", lookup.Fields))); lookup.Refresh(); } var itemlist = new List(); var iRow = 0; while (MoveNext()) { iRow++; var item = new T(); Notify(string.Format("Parsing Row {0}", iRow)); Dictionary values; try { var lineValues = ReadLine(); values = Mappings.ToDictionary( x => x.Property, x => { if (!x.Field.IsNullOrWhiteSpace()) { return lineValues.GetValueOrDefault(x.Field) ?? string.Empty; } else { return x.Constant; } }); } catch(Exception e) { WriteError(iRow, $"Error reading line: {e.Message}"); continue; } var ok = BeforeProcess == null || BeforeProcess.Invoke(this, values); if (!ok) { continue; } // First import lookups foreach(var mapping in Mappings.Where(x => x.Lookup != ImportLookupType.None)) { var value = values.GetValueOrDefault(mapping.Property) ?? string.Empty; try { ImportValues(item, mapping, value); } catch (Exception e) { WriteError(iRow, string.Format("Unable to Set Value: {0} [{1}] {2}", mapping.Property, value, e.Message)); ok = false; break; } try { DoLookup(item, mapping, lookups); } catch (Exception e) { ok = false; WriteError(iRow, string.Format("Exception setting lookup values: {0}", e.Message)); break; } } if (!ok) { continue; } // Then do non-lookups foreach (var mapping in Mappings.Where(x => x.Lookup == ImportLookupType.None)) { var value = values.GetValueOrDefault(mapping.Property) ?? string.Empty; try { ImportValues(item, mapping, value); } catch (Exception e) { WriteError(iRow, string.Format("Unable to Set Value: {0} [{1}] {2}", mapping.Property, value, e.Message)); ok = false; break; } } if (!ok) { continue; } // Do Primary key lookup if (keylookup != null) { try { var keyrows = keylookup.Results.Rows.ToList(); foreach (var mapping in Mappings.Where(x => x.Key)) { var keyvalue = CoreUtils.GetPropertyValue(item, mapping.Property); keyrows = keyrows.Where(r => string.Equals(r.Get(mapping.Property)?.Trim(), keyvalue?.ToString().Trim())) .ToList(); } var keyid = keyrows.Any() ? keyrows.First().Get("ID") : Guid.Empty; CoreUtils.SetPropertyValue(item, "ID", keyid); } catch (Exception e) { ok = false; WriteError(iRow, string.Format("Unable to set Primary Key: {0}", e.Message)); } } else { CoreUtils.SetPropertyValue(item, "ID", Guid.Empty); } if (!ok) { continue; } var bOK = AfterProcess == null || AfterProcess.Invoke(this, item, values); if (bOK && item.IsChanged()) { try { var bNewKey = keylookup != null && item.ID == Guid.Empty; if (OnSave != null) OnSave?.Invoke(this, item); else new Client().Save(item, ""); if (bNewKey) { keylookup!.Results.LoadRow(item); } var key = new List(); foreach (var mapping in Mappings.Where(x => x.Key)) key.Add(CoreUtils.GetPropertyValue(item, mapping.Property)); WriteLog(iRow, string.Format("Successfully Imported [{0}]", string.Join(" + ", key))); iResult++; } catch (Exception e) { WriteError(iRow, string.Format("Unable to Save Item: {0}", e.Message)); } } } _log.Add(""); return iResult; } protected void Notify(string message) { OnNotify?.Invoke(this, message); } private void WriteLog(int row, string message) { _log.Add($"{row:D8} {message}"); OnLog?.Invoke(this, $"{row:D8} {message}"); } private void WriteError(int row, string message) { _log.Add($"{row:D8} Error: {message}"); OnLog?.Invoke(this, $"{row:D8} Error: {message}"); OnError?.Invoke(this, $"{row:D8} {message}"); } private class ImportLookup { public ImportLookup(Type type, string id, ImportLoadEvent onload) { Type = type; Fields = new List(); ID = id; OnLoad = onload; } public Type Type { get; } public List Fields { get; } public string ID { get; } public event ImportLoadEvent OnLoad; public CoreTable Results { get; private set; } public void Refresh() { if (!Fields.Contains("ID")) Fields.Add("ID"); if (OnLoad != null) Results = OnLoad?.Invoke(this, Type, Fields.ToArray(), ID); else { var client = ClientFactory.CreateClient(Type); var columns = Columns.None(Type).Add(Fields).Add("ID"); Results = client.Query(null, columns); } } } } }