inabox/InABox.Core/Imports/BaseImporter.cs

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using InABox.Clients;

namespace InABox.Core
{
    public abstract class BaseImporter<T> : IImporter, IDisposable where T : Entity, IRemotable, IPersistent, new()
    {
        private readonly List<string> _log = new List<string>();

        public BaseImporter()
        {
            Properties = CoreUtils.PropertyList(typeof(T), x => true, true).Keys.ToArray();
            Fields = new string[] { };
            Mappings = new List<ImportMapping>();
            HasHeader = true;
            HeaderRow = 1;
        }

        public void Dispose()
        {
            Close();
            Properties = null;
            Fields = null;
            Mappings = null;
        }

        public string[] Properties { get; private set; }

        public bool HasHeader { get; set; }

        public int HeaderRow { get; set; }

        public abstract bool ReadHeader();

        public string[] Fields { get; protected set; }

        public List<ImportMapping> Mappings { get; private set; }
        public IEnumerable<string> Log => _log;

        public event ImportNotificationEvent OnNotify;

        public abstract bool Open(Stream stream);

        public abstract void Close();

        public abstract bool MoveNext();

        public abstract Dictionary<string, string> ReadLine();

        public event ImportPreProcessEvent BeforeProcess;

        public event ImportPostProcessEvent AfterProcess;

        public int Import()
        {
            _log.Clear();
            _log.Add(string.Format("Import Log {0:dd/MM/yyyy hh:mm:ss}", DateTime.Now));
            _log.Add("==============================");

            var iResult = 0;

            ImportLookup? keylookup = null;

            var lookups = new List<ImportLookup>();

            Notify("Preparing Import Mappings...");

            foreach (var mapping in Mappings)
            {
                if (mapping.Key)
                {
                    if (keylookup == null)
                    {
                        keylookup = new ImportLookup(typeof(T), "ID");
                        var others = LookupFactory.DefineColumns(typeof(T));
                        foreach (var other in others.ColumnNames())
                            keylookup.Fields.Add(other);
                        lookups.Add(keylookup);
                    }

                    if (!keylookup.Fields.Contains(mapping.Property))
                        keylookup.Fields.Add(mapping.Property);
                }

                if (mapping.Lookup != ImportLookupType.None)
                {
                    var parentprop = string.Join(".", mapping.Property.Split('.').Reverse().Skip(1).Reverse());
                    var parent = CoreUtils.GetProperty(typeof(T), parentprop);
                    var childprop = mapping.Property.Split('.').Last();

                    var bt = parent.PropertyType.BaseType;
                    if (bt != null)
                    {
                        var lookuptype = bt.GetGenericArguments().FirstOrDefault();
                        var lookup = lookups.FirstOrDefault(x => x.Type == lookuptype);
                        if (lookup == null)
                        {
                            lookup = new ImportLookup(lookuptype, parentprop + ".ID");
                            var others = LookupFactory.DefineColumns(lookuptype);
                            foreach (var other in others.ColumnNames())
                                lookup.Fields.Add(other);
                            lookups.Add(lookup);
                        }

                        if (!lookup.Fields.Contains(childprop))
                            lookup.Fields.Add(childprop);
                    }
                }
            }

            if (keylookup == null)
            {
                WriteLog(0, "WARNING: No Key Fields Found - All Items will be treated as new");
                //return iResult;
            }
            else
            {
                Notify(string.Format("Loading {0} Keys [{1}]", keylookup.Type.Name, string.Join("]+[", keylookup.Fields)));
                keylookup.Refresh();
            }

            foreach (var lookup in lookups)
                if (lookup.Results == null)
                {
                    Notify(string.Format("Loading {0} Lookups [{1}]", lookup.Type.Name, string.Join("]+[", lookup.Fields)));
                    lookup.Refresh();
                }

            var itemlist = new List<T>();
            var iRow = 0;
            while (MoveNext())
            {
                iRow++;

                var item = new T();
                Notify(string.Format("Parsing Row {0}", iRow));
                var values = ReadLine();

                var bUpdatesOK = BeforeProcess != null ? BeforeProcess.Invoke(this, values) : true;
                try
                {
                    foreach (var property in values.Keys)
                    {
                        var value = values[property];
                        var prop = CoreUtils.GetProperty(typeof(T), property);
                        var type = prop.PropertyType;

                        item.OriginalValues[property] = CoreUtils.GetPropertyValue(item, property);

                        if (type == typeof(string))
                        {
                            if (prop.GetEditor() is UniqueCodeEditor || prop.GetEditor() is CodeEditor)
                                CoreUtils.SetPropertyValue(item, property, value?.ToUpper());
                            else
                                CoreUtils.SetPropertyValue(item, property, value);
                        }
                        else
                        {
                            if (string.IsNullOrWhiteSpace(value))
                            {
                                var def = type.GetDefault();
                                CoreUtils.SetPropertyValue(item, property, def);
                            }
                            else
                            {
                                var converter = TypeDescriptor.GetConverter(type);
                                try
                                {
                                    var converted = converter.ConvertFrom(value);
                                    CoreUtils.SetPropertyValue(item, property, converted);
                                }
                                catch (Exception e)
                                {
                                    WriteLog(iRow, string.Format("Unable to Set Value: {0} [{1}] {2}", property, value, e.Message));
                                }
                            }
                        }
                    }
                }
                catch (Exception e)
                {
                    bUpdatesOK = false;
                    WriteLog(iRow, string.Format("Unable to Update Values: {0}", e.Message));
                }

                if (bUpdatesOK)
                {
                    var bLookupsOK = true;

                    if (keylookup != null)
                    {
                        try
                        {
                            var keyrows = keylookup.Results.Rows.ToList();
                            foreach (var mapping in Mappings.Where(x => x.Key))
                            {
                                var keyvalue = CoreUtils.GetPropertyValue(item, mapping.Property);
                                keyrows = keyrows.Where(r => string.Equals(r.Get<string>(mapping.Property)?.Trim(), keyvalue?.ToString().Trim()))
                                    .ToList();
                            }

                            var keyid = keyrows.Any() ? keyrows.First().Get<Guid>("ID") : Guid.Empty;
                            CoreUtils.SetPropertyValue(item, "ID", keyid);
                        }
                        catch (Exception e)
                        {
                            bLookupsOK = false;
                            WriteLog(iRow, string.Format("Unable to set Primary Key: {0}", e.Message));
                        }
                    }
                    else
                    {
                        CoreUtils.SetPropertyValue(item, "ID", Guid.Empty);
                    }

                    try
                    {
                        foreach (var mapping in Mappings.Where(x => x.Lookup != ImportLookupType.None))
                        {
                            var parentprop = string.Join(".", mapping.Property.Split('.').Reverse().Skip(1).Reverse());
                            var parent = CoreUtils.GetProperty(typeof(T), parentprop);
                            var childprop = mapping.Property.Split('.').Last();
                            var bt = parent.PropertyType.BaseType;
                            if (bt != null)
                            {
                                var lookuptype = bt.GetGenericArguments().FirstOrDefault();

                                var lookup = lookups.FirstOrDefault(x => x.Type.Equals(lookuptype));

                                IEnumerable<CoreRow> lookuprows = lookup.Results.Rows;
                                var lookupvalue = CoreUtils.GetPropertyValue(item, mapping.Property) as string;
                                if (!string.IsNullOrWhiteSpace(lookupvalue))
                                {
                                    lookuprows = lookuprows.Where(r => r.Get<string>(childprop).Equals(lookupvalue));
                                    var lookupid = lookuprows.Any() ? lookuprows.First().Get<Guid>("ID") : Guid.Empty;
                                    if (lookupid == Guid.Empty)
                                    {
                                        if (mapping.Lookup == ImportLookupType.Restrict)
                                        {
                                            bLookupsOK = false;
                                            WriteLog(iRow, string.Format("Lookup Value [{0}] not found", lookupvalue));
                                        }
                                        else if (mapping.Lookup == ImportLookupType.Create)
                                        {
                                            var newlookup = Activator.CreateInstance(lookuptype);
                                            CoreUtils.SetPropertyValue(newlookup, childprop, lookupvalue);
                                            ClientFactory.CreateClient(lookuptype).Save(newlookup, "Created by Import");
                                            lookupid = (Guid?)CoreUtils.GetPropertyValue(newlookup, "ID") ?? Guid.Empty;
                                            var newrow = lookup.Results.NewRow();
                                            lookup.Results.LoadRow(newrow, newlookup);
                                            lookup.Results.Rows.Add(newrow);
                                            CoreUtils.SetPropertyValue(item, lookup.ID, lookupid);
                                            var prefix = String.Join(".", lookup.ID.Split('.').Reverse().Skip(1).Reverse());
                                            foreach (var field in lookup.Fields)
                                                CoreUtils.SetPropertyValue(item, String.Join(".", new String[] { prefix, field }), newrow[field]);

                                        }
                                    }
                                    else
                                    {
                                        CoreUtils.SetPropertyValue(item, lookup.ID, lookupid);
                                        var prefix = String.Join(".", lookup.ID.Split('.').Reverse().Skip(1).Reverse());
                                        foreach (var field in lookup.Fields)
                                            CoreUtils.SetPropertyValue(item, String.Join(".", new String[] { prefix, field }), lookuprows.First()[field]);
                                    }
                                }
                            }
                        }
                    }
                    catch (Exception e)
                    {
                        bLookupsOK = false;
                        WriteLog(iRow, string.Format("Exception setting lookup values: {0}", e.Message));
                    }

                    if (bLookupsOK)
                    {
                        var bOK = AfterProcess != null ? AfterProcess.Invoke(this, item, values) : true;

                        if (bOK && item.IsChanged())
                            try
                            {
                                var bNewKey = keylookup != null && item.ID == Guid.Empty;

                                new Client<T>().Save(item, "");

                                if (bNewKey)
                                {
                                    var row = keylookup!.Results.NewRow();
                                    keylookup.Results.LoadRow(row, item);
                                    keylookup.Results.Rows.Add(row);
                                }

                                var key = new List<object?>();
                                foreach (var mapping in Mappings.Where(x => x.Key))
                                    key.Add(CoreUtils.GetPropertyValue(item, mapping.Property));
                                WriteLog(iRow, string.Format("Successfully Imported [{0}]", string.Join(" + ", key)));

                                iResult++;
                            }
                            catch (Exception e)
                            {
                                WriteLog(iRow, string.Format("Unable to Save Item: {0}", e.Message));
                            }
                    }
                }
            }

            _log.Add("");
            return iResult;
        }

        protected void Notify(string message)
        {
            OnNotify?.Invoke(this, message);
        }

        private void WriteLog(int row, string message)
        {
            _log.Add(string.Format("{0:D8} {1}", row, message));
        }

        private class ImportLookup
        {
            public ImportLookup(Type type, string id)
            {
                Type = type;
                Fields = new List<string>();
                ID = id;
            }

            public Type Type { get; }
            public List<string> Fields { get; }
            public string ID { get; }

            public CoreTable Results { get; private set; }

            public void Refresh()
            {
                var client = ClientFactory.CreateClient(Type);
                var columns = Columns.Create(Type);
                foreach (var field in Fields)
                    columns.Add(field);
                if (!columns.ColumnNames().Contains("ID"))
                    columns.Add("ID");
                Results = client.Query(null, columns);
            }
        }
    }
}