inabox/InABox.Core/Imports/BaseImporter.cs

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using InABox.Clients;
using Newtonsoft.Json.Linq;

namespace InABox.Core
{
    public abstract class BaseImporter<T> : IImporter, IDisposable where T : Entity, IRemotable, IPersistent, new()
    {
        private readonly List<string> _log = new List<string>();

        public BaseImporter()
        {
            Properties = CoreUtils.PropertyList(typeof(T), x => true, true).Keys.ToArray();
            Fields = new string[] { };
            Mappings = new List<ImportMapping>();
            HasHeader = true;
            HeaderRow = 1;
        }

        public void Dispose()
        {
            Close();
            Properties = null;
            Fields = null;
            Mappings = null;
        }

        public string[] Properties { get; private set; }

        public bool HasHeader { get; set; }

        public int HeaderRow { get; set; }

        public abstract bool ReadHeader();

        public string[] Fields { get; protected set; }

        public List<ImportMapping> Mappings { get; private set; }

        public IEnumerable<string> Log => _log;

        public event ImportNotificationEvent OnNotify;

        public void InitialiseFields(string[] fields)
        {
            Fields = fields;
        }

        public abstract bool Open(Stream stream);

        public abstract void Close();

        public abstract bool MoveNext();

        public abstract Dictionary<string, string> ReadLine();

        public event ImportPreProcessEvent BeforeProcess;

        public event ImportPostProcessEvent AfterProcess;

        public event ImportLoadEvent OnLoad;

        public event ImportSaveEvent OnSave;

        public event ImportLogEvent OnLog;
        public event ImportLogEvent OnError;

        private void ImportValues(T item, ImportMapping mapping, string value)
        {
            var p2 = DatabaseSchema.Property(typeof(T), mapping.Property);
            //var prop = CoreUtils.GetProperty(typeof(T), property);
            var type = p2.PropertyType;

            item.OriginalValueList[mapping.Property] = CoreUtils.GetPropertyValue(item, mapping.Property);

            if (type == typeof(string))
            {
                if (p2.Editor is UniqueCodeEditor || p2.Editor is CodeEditor)
                    CoreUtils.SetPropertyValue(item, mapping.Property, value?.ToUpper());
                else
                    CoreUtils.SetPropertyValue(item, mapping.Property, value);
            }
            else
            {
                if (string.IsNullOrWhiteSpace(value))
                {
                    var def = type.GetDefault();
                    CoreUtils.SetPropertyValue(item, mapping.Property, def);
                }
                else
                {
                    var converter = TypeDescriptor.GetConverter(type);
                    var converted = converter.ConvertFrom(value);
                    CoreUtils.SetPropertyValue(item, mapping.Property, converted);
                }
            }
        }

        private void DoLookup(T item, ImportMapping mapping, List<ImportLookup> lookups)
        {
            var parentprop = string.Join(".", mapping.Property.Split('.').Reverse().Skip(1).Reverse());
            var parent = CoreUtils.GetProperty(typeof(T), parentprop);
            var childprop = mapping.Property.Split('.').Last();
            var bt = parent.PropertyType.BaseType;
            if (bt != null)
            {
                var lookuptype = bt.GetGenericArguments().FirstOrDefault();

                var lookup = lookups.FirstOrDefault(x => x.Type.Equals(lookuptype));

                IEnumerable<CoreRow> lookuprows = lookup.Results.Rows;
                var lookupvalue = CoreUtils.GetPropertyValue(item, mapping.Property) as string;
                if (!string.IsNullOrWhiteSpace(lookupvalue))
                {
                    lookuprows = lookuprows.Where(r => r.Get<string>(childprop).Equals(lookupvalue));
                    var lookupid = lookuprows.Any() ? lookuprows.First().Get<Guid>("ID") : Guid.Empty;
                    if (lookupid == Guid.Empty)
                    {
                        if (mapping.Lookup == ImportLookupType.Restrict)
                        {
                            throw new Exception(string.Format("Lookup Value [{0}] not found", lookupvalue));
                        }
                        else if (mapping.Lookup == ImportLookupType.Create)
                        {
                            var newlookup = Activator.CreateInstance(lookuptype);
                            CoreUtils.SetPropertyValue(newlookup, childprop, lookupvalue);
                            ClientFactory.CreateClient(lookuptype).Save(newlookup, "Created by Import");
                            lookupid = (Guid?)CoreUtils.GetPropertyValue(newlookup, "ID") ?? Guid.Empty;
                            var newrow = lookup.Results.NewRow();
                            lookup.Results.FillRow(newrow, newlookup);
                            lookup.Results.Rows.Add(newrow);
                            CoreUtils.SetPropertyValue(item, lookup.ID, lookupid);
                            var prefix = String.Join(".", lookup.ID.Split('.').Reverse().Skip(1).Reverse());
                            foreach (var field in lookup.Fields)
                                CoreUtils.SetPropertyValue(item, String.Join(".", new String[] { prefix, field }), newrow[field]);

                        }
                    }
                    else
                    {
                        CoreUtils.SetPropertyValue(item, lookup.ID, lookupid);
                        var prefix = String.Join(".", lookup.ID.Split('.').Reverse().Skip(1).Reverse());
                        foreach (var field in lookup.Fields)
                            CoreUtils.SetPropertyValue(item, String.Join(".", new String[] { prefix, field }), lookuprows.First()[field]);
                    }
                }
            }
        }

        public int Import()
        {
            _log.Clear();
            _log.Add(string.Format("Import Log {0:dd/MM/yyyy hh:mm:ss}", DateTime.Now));
            _log.Add("==============================");

            var iResult = 0;

            ImportLookup? keylookup = null;

            var lookups = new List<ImportLookup>();

            Notify("Preparing Import Mappings...");

            foreach (var mapping in Mappings)
            {
                if (mapping.Key)
                {
                    if (keylookup == null)
                    {
                        keylookup = new ImportLookup(typeof(T), "ID", OnLoad);
                        var others = LookupFactory.DefineColumns(typeof(T));
                        foreach (var other in others.ColumnNames())
                            keylookup.Fields.Add(other);
                        lookups.Add(keylookup);
                    }

                    if (!keylookup.Fields.Contains(mapping.Property))
                        keylookup.Fields.Add(mapping.Property);
                }

                if (mapping.Lookup != ImportLookupType.None)
                {
                    var parentprop = string.Join(".", mapping.Property.Split('.').Reverse().Skip(1).Reverse());
                    var parent = CoreUtils.GetProperty(typeof(T), parentprop);
                    var childprop = mapping.Property.Split('.').Last();

                    var bt = parent.PropertyType.BaseType;
                    if (bt != null)
                    {
                        var lookuptype = bt.GetGenericArguments().FirstOrDefault();
                        var lookup = lookups.FirstOrDefault(x => x.Type == lookuptype);
                        if (lookup == null)
                        {
                            lookup = new ImportLookup(lookuptype, parentprop + ".ID", null);
                            var others = LookupFactory.DefineColumns(lookuptype);
                            foreach (var other in others.ColumnNames())
                                lookup.Fields.Add(other);
                            lookups.Add(lookup);
                        }

                        if (!lookup.Fields.Contains(childprop))
                            lookup.Fields.Add(childprop);
                    }
                }
            }

            if (keylookup == null)
            {
                WriteLog(0, "WARNING: No Key Fields Found - All Items will be treated as new");
                //return iResult;
            }
            else
            {
                Notify(string.Format("Loading {0} Keys [{1}]", keylookup.Type.Name, string.Join("]+[", keylookup.Fields)));
                keylookup.Refresh();
            }

            foreach (var lookup in lookups)
                if (lookup.Results == null)
                {
                    Notify(string.Format("Loading {0} Lookups [{1}]", lookup.Type.Name, string.Join("]+[", lookup.Fields)));
                    lookup.Refresh();
                }

            var itemlist = new List<T>();
            var iRow = 0;
            while (MoveNext())
            {
                iRow++;

                var item = new T();
                Notify(string.Format("Parsing Row {0}", iRow));
                Dictionary<string, string> values;
                try
                {
                    var lineValues = ReadLine();
                    values = Mappings.ToDictionary(
                        x => x.Property,
                        x =>
                        {
                            if (!x.Field.IsNullOrWhiteSpace())
                            {
                                return lineValues.GetValueOrDefault(x.Field) ?? string.Empty;
                            }
                            else
                            {
                                return x.Constant;
                            }
                        });
                }
                catch(Exception e)
                {
                    WriteError(iRow, $"Error reading line: {e.Message}");
                    continue;
                }

                var ok = BeforeProcess == null || BeforeProcess.Invoke(this, values);
                if (!ok)
                {
                    continue;
                }

                // First import lookups
                foreach(var mapping in Mappings.Where(x => x.Lookup != ImportLookupType.None))
                {
                    var value = values.GetValueOrDefault(mapping.Property) ?? string.Empty;
                    try
                    {
                        ImportValues(item, mapping, value);
                    }
                    catch (Exception e)
                    {
                        WriteError(iRow, string.Format("Unable to Set Value: {0} [{1}] {2}", mapping.Property, value, e.Message));
                        ok = false;
                        break;
                    }

                    try
                    {
                        DoLookup(item, mapping, lookups);
                    }
                    catch (Exception e)
                    {
                        ok = false;
                        WriteError(iRow, string.Format("Exception setting lookup values: {0}", e.Message));
                        break;
                    }
                }
                if (!ok)
                {
                    continue;
                }

                // Then do non-lookups
                foreach (var mapping in Mappings.Where(x => x.Lookup == ImportLookupType.None))
                {
                    var value = values.GetValueOrDefault(mapping.Property) ?? string.Empty;
                    try
                    {
                        ImportValues(item, mapping, value);
                    }
                    catch (Exception e)
                    {
                        WriteError(iRow, string.Format("Unable to Set Value: {0} [{1}] {2}", mapping.Property, value, e.Message));
                        ok = false;
                        break;
                    }
                }
                if (!ok)
                {
                    continue;
                }

                // Do Primary key lookup
                if (keylookup != null)
                {
                    try
                    {
                        var keyrows = keylookup.Results.Rows.ToList();
                        foreach (var mapping in Mappings.Where(x => x.Key))
                        {
                            var keyvalue = CoreUtils.GetPropertyValue(item, mapping.Property);
                            keyrows = keyrows.Where(r => string.Equals(r.Get<string>(mapping.Property)?.Trim(), keyvalue?.ToString().Trim()))
                                .ToList();
                        }

                        var keyid = keyrows.Any() ? keyrows.First().Get<Guid>("ID") : Guid.Empty;
                        CoreUtils.SetPropertyValue(item, "ID", keyid);
                    }
                    catch (Exception e)
                    {
                        ok = false;
                        WriteError(iRow, string.Format("Unable to set Primary Key: {0}", e.Message));
                    }
                }
                else
                {
                    CoreUtils.SetPropertyValue(item, "ID", Guid.Empty);
                }
                if (!ok)
                {
                    continue;
                }

                var bOK = AfterProcess == null || AfterProcess.Invoke(this, item, values);

                if (bOK && item.IsChanged())
                {
                    try
                    {
                        var bNewKey = keylookup != null && item.ID == Guid.Empty;

                        if (OnSave != null)
                            OnSave?.Invoke(this, item);
                        else
                            new Client<T>().Save(item, "");

                        if (bNewKey)
                        {
                            keylookup!.Results.LoadRow(item);
                        }

                        var key = new List<object?>();
                        foreach (var mapping in Mappings.Where(x => x.Key))
                            key.Add(CoreUtils.GetPropertyValue(item, mapping.Property));
                        WriteLog(iRow, string.Format("Successfully Imported [{0}]", string.Join(" + ", key)));

                        iResult++;
                    }
                    catch (Exception e)
                    {
                        WriteError(iRow, string.Format("Unable to Save Item: {0}", e.Message));
                    }
                }
            }

            _log.Add("");
            return iResult;
        }

        protected void Notify(string message)
        {
            OnNotify?.Invoke(this, message);
        }

        private void WriteLog(int row, string message)
        {
            _log.Add($"{row:D8} {message}");
            OnLog?.Invoke(this, $"{row:D8} {message}");
        }

        private void WriteError(int row, string message)
        {
            _log.Add($"{row:D8} Error: {message}");
            OnLog?.Invoke(this, $"{row:D8} Error: {message}");
            OnError?.Invoke(this, $"{row:D8} {message}");
        }

        private class ImportLookup
        {
            public ImportLookup(Type type, string id, ImportLoadEvent onload)
            {
                Type = type;
                Fields = new List<string>();
                ID = id;
                OnLoad = onload;
            }

            public Type Type { get; }
            public List<string> Fields { get; }
            public string ID { get; }

            public event ImportLoadEvent OnLoad;
            
            public CoreTable Results { get; private set; }
            
            public void Refresh()
            {
                if (!Fields.Contains("ID"))
                    Fields.Add("ID");
                if (OnLoad != null)
                    Results = OnLoad?.Invoke(this, Type, Fields.ToArray(), ID);
                else
                {
                    var client = ClientFactory.CreateClient(Type);
                    var columns = Columns.None(Type).Add(Fields).Add("ID");
                    Results = client.Query(null, columns);
                }
            }
        }
    }
}