inabox/InABox.Core/Imports/BaseImporter.cs

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.IO;
using System.Linq;
using InABox.Clients;
using Newtonsoft.Json.Linq;

namespace InABox.Core
{
    public abstract class BaseImporter<T> : IImporter, IDisposable where T : Entity, IRemotable, IPersistent, new()
    {
        private readonly List<string> _log = new List<string>();

        public BaseImporter()
        {
            Properties = CoreUtils.PropertyList(typeof(T), x => true, true).Keys.ToArray();
            Fields = new string[] { };
            Mappings = new List<ImportMapping>();
            HasHeader = true;
            HeaderRow = 1;
        }

        public void Dispose()
        {
            Close();
            Properties = null;
            Fields = null;
            Mappings = null;
        }

        public string[] Properties { get; private set; }

        public bool HasHeader { get; set; }

        public int HeaderRow { get; set; }

        public abstract bool ReadHeader();

        public string[] Fields { get; protected set; }

        public List<ImportMapping> Mappings { get; private set; }

        public IEnumerable<string> Log => _log;

        public event ImportNotificationEvent OnNotify;

        public void InitialiseFields(string[] fields)
        {
            Fields = fields;
        }

        public abstract bool Open(Stream stream);

        public abstract void Close();

        public abstract bool MoveNext();

        public abstract Dictionary<string, string> ReadLine();

        public event ImportPreProcessEvent BeforeProcess;

        public event ImportPostProcessEvent AfterProcess;

        public event ImportLoadEvent OnLoad;

        public event ImportSaveEvent OnSave;

        public event ImportLogEvent OnLog;
        public event ImportLogEvent OnError;

        public int Import()
        {
            _log.Clear();
            _log.Add(string.Format("Import Log {0:dd/MM/yyyy hh:mm:ss}", DateTime.Now));
            _log.Add("==============================");

            var iResult = 0;

            ImportLookup? keylookup = null;

            var lookups = new List<ImportLookup>();

            Notify("Preparing Import Mappings...");

            foreach (var mapping in Mappings)
            {
                if (mapping.Key)
                {
                    if (keylookup == null)
                    {
                        keylookup = new ImportLookup(typeof(T), "ID", OnLoad);
                        var others = LookupFactory.DefineColumns(typeof(T));
                        foreach (var other in others.ColumnNames())
                            keylookup.Fields.Add(other);
                        lookups.Add(keylookup);
                    }

                    if (!keylookup.Fields.Contains(mapping.Property))
                        keylookup.Fields.Add(mapping.Property);
                }

                if (mapping.Lookup != ImportLookupType.None)
                {
                    var parentprop = string.Join(".", mapping.Property.Split('.').Reverse().Skip(1).Reverse());
                    var parent = CoreUtils.GetProperty(typeof(T), parentprop);
                    var childprop = mapping.Property.Split('.').Last();

                    var bt = parent.PropertyType.BaseType;
                    if (bt != null)
                    {
                        var lookuptype = bt.GetGenericArguments().FirstOrDefault();
                        var lookup = lookups.FirstOrDefault(x => x.Type == lookuptype);
                        if (lookup == null)
                        {
                            lookup = new ImportLookup(lookuptype, parentprop + ".ID", null);
                            var others = LookupFactory.DefineColumns(lookuptype);
                            foreach (var other in others.ColumnNames())
                                lookup.Fields.Add(other);
                            lookups.Add(lookup);
                        }

                        if (!lookup.Fields.Contains(childprop))
                            lookup.Fields.Add(childprop);
                    }
                }
            }

            if (keylookup == null)
            {
                WriteLog(0, "WARNING: No Key Fields Found - All Items will be treated as new");
                //return iResult;
            }
            else
            {
                Notify(string.Format("Loading {0} Keys [{1}]", keylookup.Type.Name, string.Join("]+[", keylookup.Fields)));
                keylookup.Refresh();
            }

            foreach (var lookup in lookups)
                if (lookup.Results == null)
                {
                    Notify(string.Format("Loading {0} Lookups [{1}]", lookup.Type.Name, string.Join("]+[", lookup.Fields)));
                    lookup.Refresh();
                }

            var itemlist = new List<T>();
            var iRow = 0;
            while (MoveNext())
            {
                iRow++;

                var item = new T();
                Notify(string.Format("Parsing Row {0}", iRow));
                Dictionary<string, string> values;
                try
                {
                    var lineValues = ReadLine();
                    values = Mappings.ToDictionary(
                        x => x.Property,
                        x =>
                        {
                            if (!x.Field.IsNullOrWhiteSpace())
                            {
                                return lineValues.GetValueOrDefault(x.Field) ?? "";
                            }
                            else
                            {
                                return x.Constant;
                            }
                        });
                }
                catch(Exception e)
                {
                    WriteError(iRow, $"Error reading line: {e.Message}");
                    continue;
                }

                var bUpdatesOK = BeforeProcess != null ? BeforeProcess.Invoke(this, values) : true;
                try
                {
                    foreach (var property in values.Keys)
                    {
                        var value = values[property];
                        var p2 = DatabaseSchema.Property(typeof(T), property);
                        //var prop = CoreUtils.GetProperty(typeof(T), property);
                        var type = p2.PropertyType;

                        item.OriginalValues[property] = CoreUtils.GetPropertyValue(item, property);

                        if (type == typeof(string))
                        {
                            if (p2.Editor is UniqueCodeEditor || p2.Editor is CodeEditor)
                                CoreUtils.SetPropertyValue(item, property, value?.ToUpper());
                            else
                                CoreUtils.SetPropertyValue(item, property, value);
                        }
                        else
                        {
                            if (string.IsNullOrWhiteSpace(value))
                            {
                                var def = type.GetDefault();
                                CoreUtils.SetPropertyValue(item, property, def);
                            }
                            else
                            {
                                var converter = TypeDescriptor.GetConverter(type);
                                try
                                {
                                    var converted = converter.ConvertFrom(value);
                                    CoreUtils.SetPropertyValue(item, property, converted);
                                }
                                catch (Exception e)
                                {
                                    WriteError(iRow, string.Format("Unable to Set Value: {0} [{1}] {2}", property, value, e.Message));
                                }
                            }
                        }
                    }
                }
                catch (Exception e)
                {
                    bUpdatesOK = false;
                    WriteError(iRow, string.Format("Unable to Update Values: {0}", e.Message));
                }
                if (!bUpdatesOK)
                {
                    continue;
                }

                var bLookupsOK = true;

                if (keylookup != null)
                {
                    try
                    {
                        var keyrows = keylookup.Results.Rows.ToList();
                        foreach (var mapping in Mappings.Where(x => x.Key))
                        {
                            var keyvalue = CoreUtils.GetPropertyValue(item, mapping.Property);
                            keyrows = keyrows.Where(r => string.Equals(r.Get<string>(mapping.Property)?.Trim(), keyvalue?.ToString().Trim()))
                                .ToList();
                        }

                        var keyid = keyrows.Any() ? keyrows.First().Get<Guid>("ID") : Guid.Empty;
                        CoreUtils.SetPropertyValue(item, "ID", keyid);
                    }
                    catch (Exception e)
                    {
                        bLookupsOK = false;
                        WriteError(iRow, string.Format("Unable to set Primary Key: {0}", e.Message));
                    }
                }
                else
                {
                    CoreUtils.SetPropertyValue(item, "ID", Guid.Empty);
                }

                try
                {
                    foreach (var mapping in Mappings.Where(x => x.Lookup != ImportLookupType.None))
                    {
                        var parentprop = string.Join(".", mapping.Property.Split('.').Reverse().Skip(1).Reverse());
                        var parent = CoreUtils.GetProperty(typeof(T), parentprop);
                        var childprop = mapping.Property.Split('.').Last();
                        var bt = parent.PropertyType.BaseType;
                        if (bt != null)
                        {
                            var lookuptype = bt.GetGenericArguments().FirstOrDefault();

                            var lookup = lookups.FirstOrDefault(x => x.Type.Equals(lookuptype));

                            IEnumerable<CoreRow> lookuprows = lookup.Results.Rows;
                            var lookupvalue = CoreUtils.GetPropertyValue(item, mapping.Property) as string;
                            if (!string.IsNullOrWhiteSpace(lookupvalue))
                            {
                                lookuprows = lookuprows.Where(r => r.Get<string>(childprop).Equals(lookupvalue));
                                var lookupid = lookuprows.Any() ? lookuprows.First().Get<Guid>("ID") : Guid.Empty;
                                if (lookupid == Guid.Empty)
                                {
                                    if (mapping.Lookup == ImportLookupType.Restrict)
                                    {
                                        bLookupsOK = false;
                                        WriteError(iRow, string.Format("Lookup Value [{0}] not found", lookupvalue));
                                    }
                                    else if (mapping.Lookup == ImportLookupType.Create)
                                    {
                                        var newlookup = Activator.CreateInstance(lookuptype);
                                        CoreUtils.SetPropertyValue(newlookup, childprop, lookupvalue);
                                        ClientFactory.CreateClient(lookuptype).Save(newlookup, "Created by Import");
                                        lookupid = (Guid?)CoreUtils.GetPropertyValue(newlookup, "ID") ?? Guid.Empty;
                                        var newrow = lookup.Results.NewRow();
                                        lookup.Results.LoadRow(newrow, newlookup);
                                        lookup.Results.Rows.Add(newrow);
                                        CoreUtils.SetPropertyValue(item, lookup.ID, lookupid);
                                        var prefix = String.Join(".", lookup.ID.Split('.').Reverse().Skip(1).Reverse());
                                        foreach (var field in lookup.Fields)
                                            CoreUtils.SetPropertyValue(item, String.Join(".", new String[] { prefix, field }), newrow[field]);

                                    }
                                }
                                else
                                {
                                    CoreUtils.SetPropertyValue(item, lookup.ID, lookupid);
                                    var prefix = String.Join(".", lookup.ID.Split('.').Reverse().Skip(1).Reverse());
                                    foreach (var field in lookup.Fields)
                                        CoreUtils.SetPropertyValue(item, String.Join(".", new String[] { prefix, field }), lookuprows.First()[field]);
                                }
                            }
                        }
                    }
                }
                catch (Exception e)
                {
                    bLookupsOK = false;
                    WriteError(iRow, string.Format("Exception setting lookup values: {0}", e.Message));
                }

                if (bLookupsOK)
                {
                    var bOK = AfterProcess != null ? AfterProcess.Invoke(this, item, values) : true;

                    if (bOK && item.IsChanged())
                        try
                        {
                            var bNewKey = keylookup != null && item.ID == Guid.Empty;

                            if (OnSave != null)
                                OnSave?.Invoke(this, item);
                            else
                                new Client<T>().Save(item, "");

                            if (bNewKey)
                            {
                                var row = keylookup!.Results.NewRow();
                                keylookup.Results.LoadRow(row, item);
                                keylookup.Results.Rows.Add(row);
                            }

                            var key = new List<object?>();
                            foreach (var mapping in Mappings.Where(x => x.Key))
                                key.Add(CoreUtils.GetPropertyValue(item, mapping.Property));
                            WriteLog(iRow, string.Format("Successfully Imported [{0}]", string.Join(" + ", key)));

                            iResult++;
                        }
                        catch (Exception e)
                        {
                            WriteError(iRow, string.Format("Unable to Save Item: {0}", e.Message));
                        }
                }
            }

            _log.Add("");
            return iResult;
        }

        protected void Notify(string message)
        {
            OnNotify?.Invoke(this, message);
        }

        private void WriteLog(int row, string message)
        {
            _log.Add($"{row:D8} {message}");
            OnLog?.Invoke(this, $"{row:D8} {message}");
        }

        private void WriteError(int row, string message)
        {
            _log.Add($"{row:D8} Error: {message}");
            OnLog?.Invoke(this, $"{row:D8} Error: {message}");
            OnError?.Invoke(this, $"{row:D8} {message}");
        }

        private class ImportLookup
        {
            public ImportLookup(Type type, string id, ImportLoadEvent onload)
            {
                Type = type;
                Fields = new List<string>();
                ID = id;
                OnLoad = onload;
            }

            public Type Type { get; }
            public List<string> Fields { get; }
            public string ID { get; }

            public event ImportLoadEvent OnLoad;
            
            public CoreTable Results { get; private set; }
            
            public void Refresh()
            {
                if (!Fields.Contains("ID"))
                    Fields.Add("ID");
                if (OnLoad != null)
                    Results = OnLoad?.Invoke(this, Type, Fields.ToArray(), ID);
                else
                {
                    var client = ClientFactory.CreateClient(Type);
                    var columns = Columns.Create(Type);
                    foreach (var field in Fields)
                        columns.Add(field);
                    if (!columns.ColumnNames().Contains("ID"))
                        columns.Add("ID");
                    Results = client.Query(null, columns);
                }
            }
        }
    }
}