Created tabular file reader interface and worked on spreadsheet interface

InABox.Core/Client/Client.cs

@@ -66,6 +66,12 @@ namespace InABox.Clients
             }
         }
 
+        public static CoreTable Query<TEntity>(Filter<TEntity>? filter = null, Columns<TEntity>? columns = null, SortOrder<TEntity>? orderby = null)
+            where TEntity : Entity, IRemotable, IPersistent, new()
+        {
+            return new Client<TEntity>().Query(filter, columns, orderby);
+        }
+
         public static void QueryMultiple(
             Action<Dictionary<string, CoreTable>?, Exception?> callback,
             Dictionary<string, IQueryDef> queries)

InABox.Core/Imports/BaseImporter.cs

@@ -45,6 +45,11 @@ namespace InABox.Core
 
         public event ImportNotificationEvent OnNotify;
 
+        public void InitialiseFields(string[] fields)
+        {
+            Fields = fields;
+        }
+
         public abstract bool Open(Stream stream);
 
         public abstract void Close();

inabox.scripting/FileReader/CSVFileReader.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace InABox.Scripting
+{
+    public class CSVFileReader : DelimitedFileReader
+    {
+        public CSVFileReader(Stream stream) : base(stream, ",")
+        {
+        }
+    }
+}

inabox.scripting/FileReader/DelimitedFileReader.cs

@@ -0,0 +1,68 @@
+using JetBrains.Annotations;
+using NPOI.SS.Formula.Functions;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using TextFieldParserStandard;
+
+namespace InABox.Scripting
+{
+    public class DelimitedFileReader : ITabularFileReader, IDisposable
+    {
+        private TextFieldParser _parser;
+
+        public bool EndOfData => _parser.EndOfData;
+
+        public Dictionary<string, int> Columns { get; set; }
+
+        public DelimitedFileReader(Stream stream, string delimiter)
+        {
+            _parser = new TextFieldParser(stream);
+            _parser.TextFieldType = FieldType.Delimited;
+            _parser.SetDelimiters(delimiter);
+
+            Columns = new();
+        }
+
+        public bool ReadHeader()
+        {
+            if (_parser.EndOfData)
+            {
+                return false;
+            }
+            Columns = _parser.ReadFields().Select((field, i) => (field, i)).ToDictionary(x => x.field, x => x.i);
+            return true;
+        }
+
+        public bool SkipLine()
+        {
+            if (_parser.EndOfData)
+            {
+                return false;
+            }
+            _parser.ReadLine();
+            return true;
+        }
+
+        public Dictionary<string, object?> ReadLine()
+        {
+            var values = _parser.ReadFields();
+            return Columns.ToDictionary(
+                x => x.Key,
+                x =>
+                {
+                    object? result = values[x.Value];
+                    return result;
+                });
+        }
+
+        public void Dispose()
+        {
+            _parser.Dispose();
+        }
+
+        public IList<string> ReadLineValues() => _parser.ReadFields();
+    }
+}

inabox.scripting/FileReader/ExcelFileReader.cs

@@ -0,0 +1,131 @@
+using NPOI.SS.Formula.Functions;
+using NPOI.SS.UserModel;
+using InABox.Core;
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using static NPOI.HSSF.UserModel.HeaderFooter;
+
+namespace InABox.Scripting
+{
+    public class ExcelFileReader : ITabularFileReader
+    {
+        private IEnumerator<IRow> rows;
+
+        public Dictionary<string, int> Columns { get; set; }
+
+        public bool EndOfData { get; private set; }
+
+        private IRow _row;
+
+        public ExcelFileReader(Stream stream)
+        {
+            rows = new Spreadsheet(stream).GetSheet(0).RowEnumerator();
+            Columns = new Dictionary<string, int>();
+            SkipLine();
+        }
+
+        public IList<string> ReadLineValues()
+        {
+            var results = new List<string>();
+
+            if (!EndOfData)
+            {
+                foreach (var cell in _row.Cells())
+                {
+                    results.Add(cell.GetValue());
+                }
+                SkipLine();
+            }
+            return results;
+        }
+
+        public bool ReadHeader()
+        {
+            Columns.Clear();
+
+            if (EndOfData)
+            {
+                return false;
+            }
+
+            int i = 0;
+            foreach(var cell in _row.Cells())
+            {
+                var column = cell.GetValue();
+                if (!column.IsNullOrWhiteSpace())
+                {
+                    Columns.Add(column, i);
+                }
+                ++i;
+            }
+            SkipLine();
+
+            return true;
+        }
+
+        public Dictionary<string, object?> ReadLine()
+        {
+            if (EndOfData)
+            {
+                return new Dictionary<string, object?>();
+            }
+
+            var results = Columns.ToDictionary(
+                x => x.Key,
+                x =>
+                {
+                    object? result;
+                    var cell = _row.GetCell(x.Value);
+                    if(cell is null)
+                    {
+                        result = null;
+                    }
+                    else
+                    {
+                        switch(cell.GetCellType())
+                        {
+                            case CellType.Formula:
+                                result = cell.GetValue();
+                                break;
+                            case CellType.Numeric:
+                                result = cell.GetDoubleValue();
+                                break;
+                            case CellType.Date:
+                                result = cell.GetDateTimeValue();
+                                break;
+                            case CellType.String:
+                                result = cell.GetValue();
+                                break;
+                            case CellType.Boolean:
+                                result = cell.GetBoolValue();
+                                break;
+                            default:
+                                result = null;
+                                break;
+                        }
+                    }
+                    return result;
+                });
+            SkipLine();
+            return results;
+        }
+
+        public bool SkipLine()
+        {
+            if (!EndOfData)
+            {
+                EndOfData = rows.MoveNext();
+                if (!EndOfData)
+                {
+                    _row = rows.Current;
+                }
+            }
+            return EndOfData;
+        }
+    }
+}

inabox.scripting/FileReader/ITabularFileReader.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace InABox.Scripting
+{
+    public interface ITabularFileReader
+    {
+        Dictionary<string, int> Columns { get; set; }
+
+        bool EndOfData { get; }
+
+        bool ReadHeader();
+
+        bool SkipLine();
+
+        Dictionary<string, object?> ReadLine();
+
+        IList<string> ReadLineValues();
+    }
+}

inabox.scripting/ScriptDocument.cs

@@ -87,13 +87,14 @@ namespace InABox.Scripting
         }
 
         private static MethodInfo HasSubmissionResult { get; } =
-            typeof(Compilation).GetMethod(nameof(HasSubmissionResult), BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance);
+            typeof(Compilation).GetMethod(nameof(HasSubmissionResult), BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance)
+                ?? throw new NullReferenceException();
 
         private static PrintOptions PrintOptions { get; } = new() { MemberDisplayFormat = MemberDisplayFormat.SeparateLines };
 
         public List<ScriptProperty> Properties { get; }
 
-        public event PropertyChangedEventHandler PropertyChanged;
+        public event PropertyChangedEventHandler? PropertyChanged;
 
         private static IEnumerable<MetadataReference> CompilationReferences;
 

inabox.scripting/Spreadsheet/ISheet.cs

@@ -11,6 +11,18 @@ using InABox.Core;
 namespace InABox.Scripting
 {
 
+    public enum CellType
+    {
+        Formula,
+        Numeric,
+        Error,
+        String,
+        Boolean,
+        Blank,
+        Unknown,
+        Date
+    }
+
     public class CellRange
     {
         public int FirstRow { get; set; }
@@ -161,6 +173,8 @@ namespace InABox.Scripting
     {
         IRow Row { get; }
 
+        CellType GetCellType();
+
         string GetValue();
         bool? GetBoolValue();
         double? GetDoubleValue();

inabox.scripting/Spreadsheet/NPOISpreadsheet.cs

@@ -13,12 +13,14 @@ using NFont = NPOI.SS.UserModel.IFont;
 using NSheet = NPOI.SS.UserModel.ISheet;
 using NDataFormat = NPOI.SS.UserModel.IDataFormat;
 using NCellStyle = NPOI.SS.UserModel.ICellStyle;
+using NCellType = NPOI.SS.UserModel.CellType;
 using NPOI.SS.Util;
 using NPOI.OpenXmlFormats.Spreadsheet;
 using System.Security.Policy;
 using System.Drawing;
 using NPOI.HSSF.Util;
 using NPOI.HSSF.UserModel;
+using NPOI.XWPF.UserModel;
 
 namespace InABox.Scripting
 {
@@ -164,7 +166,7 @@ namespace InABox.Scripting
             {
                 var result = "";
                 var cell = _row.GetCell(column, MissingCellPolicy.CREATE_NULL_AS_BLANK);
-                if (cell.CellType == CellType.Numeric)
+                if (cell.CellType == NCellType.Numeric)
                     result = cell.NumericCellValue.ToString();
                 else
                     result = cell.StringCellValue;
@@ -206,9 +208,9 @@ namespace InABox.Scripting
             {
                 double result = 0.0F;
                 var cell = _row.GetCell(column, MissingCellPolicy.CREATE_NULL_AS_BLANK);
-                if (cell.CellType == CellType.Numeric || cell.CellType == CellType.Formula)
+                if (cell.CellType == NCellType.Numeric || cell.CellType == NCellType.Formula)
                     result = cell.NumericCellValue;
-                else if (cell.CellType == CellType.String)
+                else if (cell.CellType == NCellType.String)
                     result = double.Parse(cell.StringCellValue);
                 return result;
             }
@@ -279,11 +281,38 @@ namespace InABox.Scripting
             Row = row;
         }
 
+        private CellType ConvertCellType(NCellType type)
+        {
+            return type switch
+            {
+                NCellType.Formula => CellType.Numeric,
+                NCellType.Numeric => CellType.Numeric,
+                NCellType.Error => CellType.Error,
+                NCellType.String => CellType.String,
+                NCellType.Boolean => CellType.Boolean,
+                NCellType.Blank => CellType.Blank,
+                _ or NCellType.Unknown => CellType.Unknown
+            };
+        }
+
+        public CellType GetCellType()
+        {
+            if (_cell.CellType == NCellType.Formula)
+            {
+                return ConvertCellType(_cell.CachedFormulaResultType);
+            }
+            else if(_cell.CellType == NCellType.Numeric && DateUtil.IsCellDateFormatted(_cell))
+            {
+                return CellType.Date;
+            }
+            return ConvertCellType(_cell.CellType);
+        }
+
         public string GetValue()
         {
-            if (_cell.CellType == CellType.Formula)
+            if (_cell.CellType == NCellType.Formula)
             {
-                if (_cell.CachedFormulaResultType == CellType.Numeric)
+                if (_cell.CachedFormulaResultType == NCellType.Numeric)
                     return string.Format("{0:F}", _cell.NumericCellValue.ToString());
                 return _cell.StringCellValue;
             }
@@ -295,7 +324,7 @@ namespace InABox.Scripting
         {
             try
             {
-                if (_cell.CellType == CellType.Boolean)
+                if (_cell.CellType == NCellType.Boolean)
                     return _cell.BooleanCellValue;
                 return null;
             }
@@ -310,9 +339,9 @@ namespace InABox.Scripting
             try
             {
                 double result = 0.0F;
-                if (_cell.CellType == CellType.Numeric || _cell.CellType == CellType.Formula)
+                if (_cell.CellType == NCellType.Numeric || _cell.CellType == NCellType.Formula)
                     result = _cell.NumericCellValue;
-                else if (_cell.CellType == CellType.String)
+                else if (_cell.CellType == NCellType.String)
                     result = double.Parse(_cell.StringCellValue);
                 return result;
             }
@@ -343,9 +372,9 @@ namespace InABox.Scripting
             try
             {
                 byte result = 0;
-                if (_cell.CellType == CellType.Numeric || _cell.CellType == CellType.Formula)
+                if (_cell.CellType == NCellType.Numeric || _cell.CellType == NCellType.Formula)
                     result = (byte)_cell.NumericCellValue;
-                else if (_cell.CellType == CellType.String)
+                else if (_cell.CellType == NCellType.String)
                     result = byte.Parse(_cell.StringCellValue);
                 return result;
             }
@@ -604,7 +633,7 @@ namespace InABox.Scripting
 
         public Spreadsheet(string fileName) : this(WorkbookFactory.Create(fileName)) { }
 
-        public Spreadsheet(FileStream file) : this(WorkbookFactory.Create(file)) { }
+        public Spreadsheet(Stream file) : this(WorkbookFactory.Create(file)) { }
 
         public Spreadsheet() : this(new XSSFWorkbook()) { }
 

inabox.wpf/DynamicGrid/DynamicImportMappingGrid.cs

@@ -68,6 +68,13 @@ namespace InABox.DynamicGrid
                 result.LoadRows(Items.Where(x => !string.IsNullOrWhiteSpace(x.Field) || !string.IsNullOrWhiteSpace(x.Constant)));
             else
                 result.LoadRows(Items);
+            foreach(var item in Items)
+            {
+                if(!item.Field.IsNullOrWhiteSpace() && !ImportFieldGenerator.Fields.Contains(item.Field))
+                {
+                    item.Field = "";
+                }
+            }
             action.Invoke(result, null);
         }