prs/prs.server/Engines/WebEngine/WebListener.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Net;
using System.Reflection;
using System.Security.Cryptography.X509Certificates;
using System.Threading.Tasks;
using Comal.Classes;
using GenHTTP.Api.Content;
using GenHTTP.Api.Infrastructure;
using GenHTTP.Api.Protocol;
using GenHTTP.Api.Routing;
using GenHTTP.Engine;
using GenHTTP.Modules.IO;
using GenHTTP.Modules.IO.Streaming;
using GenHTTP.Modules.Practices;
using InABox.Clients;
using InABox.Core;
using Newtonsoft.Json.Linq;
using PRSServer.Engines;
using RazorEngine;
using RazorEngine.Compilation;
using RazorEngine.Compilation.ReferenceResolver;
using RazorEngine.Configuration;
using RazorEngine.Templating;
using static System.Windows.Forms.VisualStyles.VisualStyleElement.StartPanel;
using Cookie = GenHTTP.Api.Protocol.Cookie;
using RequestMethod = GenHTTP.Api.Protocol.RequestMethod;

namespace PRSServer
{
    public class WebModel : IDataModel
    {
        public WebModel(DataModel model, IRequest request, User? user)
        {
            _dataModel = model;
            Request = request;
            User = user;
        }

        public User? User { get; }

        private IDataModel _dataModel { get; }

        /// <summary>
        /// A set of cookies to be used in the event of normal response.
        /// </summary>
        /// <remarks>
        /// Note that these are not used if <see cref="Respond"/> has been called.
        /// </remarks>
        public Dictionary<string, string> Cookies { get; } = new();

        public IRequest Request { get; init; }

        public IResponseBuilder? Response { get; private set; }

        public IEnumerable<System.Data.DataTable> DefaultTables => throw new NotImplementedException();

        public IResponseBuilder Respond()
        {
            Response = Request.Respond();
            return Response;
        }

        public bool UserCanAccess<T>() where T : ISecurityDescriptor, new()
        {
            return User != null && Security.IsAllowed<T>(User.ID, User.SecurityGroup.ID);
        }

        public bool IsLoggedIn()
        {
            return User != null;
        }

        public void AddTable(Type type, CoreTable table, bool isdefault = false, string? alias = null)
        {
            _dataModel.AddTable(type, table, isdefault, alias);
        }

        public void AddTable<TType>(Filter<TType>? filter, Columns<TType>? columns, bool isdefault = false, string? alias = null,
            bool shouldLoad = true)
        {
            _dataModel.AddTable(filter, columns, isdefault, alias, shouldLoad);
        }
        public void AddTable(string alias, CoreTable table, bool isdefault = false)
        {
            _dataModel.AddTable(alias, table, isdefault);
        }

        public void LinkTable(Type parenttype, string parentcolumn, Type childtype, string childcolumn, string? parentalias = null,
            string? childalias = null, bool isLookup = false)
        {
            _dataModel.LinkTable(parenttype, parentcolumn, childtype, childcolumn, parentalias, childalias);
        }

        public void LinkTable<TParent, TChild>(Expression<Func<TParent, object>> parent, Expression<Func<TChild, object>> child,
            string? parentalias = null, string? childalias = null, bool isLookup = false)
        {
            _dataModel.LinkTable(parent, child, parentalias, childalias);
        }

        public void LinkTable(Type parenttype, string parentcolumn, string childalias, string childcolumn, string? parentalias = null, bool isLookup = false)
        {
            _dataModel.LinkTable(parenttype, parentcolumn, childalias, childcolumn, parentalias, isLookup);
        }

        public void AddChildTable<TParent, TChild>(Expression<Func<TParent, object>> parentcolumn, Expression<Func<TChild, object>> childcolumn,
            Filter<TChild>? filter = null, Columns<TChild>? columns = null, bool isdefault = false, string? parentalias = null,
            string? childalias = null)
        {
            _dataModel.AddChildTable(parentcolumn, childcolumn, filter, columns, isdefault, parentalias, childalias);
        }

        public void AddLookupTable<TSource, TLookup>(Expression<Func<TSource, object>> sourcecolumn, Expression<Func<TLookup, object>> lookupcolumn,
            Filter<TLookup>? filter = null, Columns<TLookup>? columns = null, bool isdefault = false, string? sourcealias = null,
            string? lookupalias = null)
        {
            _dataModel.AddLookupTable(sourcecolumn, lookupcolumn, filter, columns, isdefault, sourcealias, lookupalias);
        }

        public CoreTable GetTable<TType>(string? alias = null)
        {
            return _dataModel.GetTable<TType>(alias);
        }

        public void SetTableData(Type type, CoreTable tableData, string? alias = null)
        {
            _dataModel.SetTableData(type, tableData, alias);
        }

        public bool HasTable(Type type, string? alias = null)
        {
            return _dataModel.HasTable(type, alias);
        }

        public bool HasTable<TType>(string? alias = null)
        {
            return _dataModel.HasTable<TType>(alias);
        }

        public void LoadModel(IEnumerable<string>? requiredTables, Dictionary<string, IQueryDef>? requiredQueries = null)
        {
            _dataModel.LoadModel(requiredTables, requiredQueries);
        }

        public void LoadModel(IEnumerable<string>? requiredTables, params IDataModelQueryDef[] requiredQueries)
        {
            _dataModel.LoadModel(requiredTables, requiredQueries);
        }
        public void LoadModel()
        {
            _dataModel.LoadModel();
        }

        public TType[] ExtractValues<TSource, TType>(Expression<Func<TSource, TType>> column, bool distinct = true, string? alias = null)
        {
            return _dataModel.ExtractValues(column, distinct, alias);
        }

        public bool RemoveTable<TType>(string? alias = null)
        {
            return _dataModel.RemoveTable<TType>(alias);
        }

        public Filter<TType>? GetFilter<TType>(string? alias = null) => _dataModel.GetFilter<TType>(alias);
        public Columns<TType>? GetColumns<TType>(string? alias = null) => _dataModel.GetColumns<TType>(alias);

        public void SetFilter<TType>(Filter<TType>? filter, string? alias = null) => _dataModel.SetFilter(filter, alias);

        public void SetColumns<TType>(Columns<TType>? columns, string? alias = null) => _dataModel.SetColumns(columns, alias);

        public void SetIsDefault<TType>(bool isDefault, string? alias = null) => _dataModel.SetIsDefault<TType>(isDefault, alias);

        public void SetShouldLoad<TType>(bool shouldLoad, string? alias = null) => _dataModel.SetShouldLoad<TType>(shouldLoad, alias);

    }

    public class RazorReferenceResolver : IReferenceResolver
    {
        public IEnumerable<CompilerReference> GetReferences(TypeContext context, IEnumerable<CompilerReference>? includeAssemblies = null)
        {
            foreach(var reference in new UseCurrentAssembliesReferenceResolver().GetReferences(context, includeAssemblies))
            {
                yield return reference;
            }
            foreach(var reference in Directory.GetFiles(AppDomain.CurrentDomain.BaseDirectory, "*.dll")
                .Where(x => !Path.GetFileName(x).StartsWith("pdfium")
                    && !Path.GetFileName(x).StartsWith("gsdll")
                    && !Path.GetFileName(x).StartsWith("ikvm-native"))
                .Select(x => CompilerReference.From(x)))
            {
                yield return reference;
            }
        }
    }

    public class WebHandler : Handler<WebHandlerProperties>
    {
        public ulong LoginExpiry; // In seconds

        public override void Init(WebHandlerProperties properties)
        {
            MaxFileSize = properties.MaxFileSize;
            LoginExpiry = properties.LoginExpiry;

            if (CoreUtils.GetVersion() != "???") CompileAllTemplates();
        }

        static WebHandler()
        {
            var config = new TemplateServiceConfiguration();
            config.BaseTemplateType = typeof(WebTemplateBase<>);
            config.ReferenceResolver = new RazorReferenceResolver();
            RazorService = RazorEngineService.Create(config);
        }

        public int MaxFileSize { get; set; }

        private void RunTestDataModel<T>()
            where T : Entity, IRemotable, IPersistent, new()
        {
            Logger.Send(LogType.Information, "", string.Format("Started '{0}'", typeof(T).Name));
            var id = new Client<T>().Query(null, new Columns<T>(x => x.ID)).ExtractValues<Guid>("ID").FirstOrDefault();

            Logger.Send(LogType.Information, "", string.Format("Using id '{0}'", id));
            var autoDataModel = new AutoDataModel<T>(new Filter<T>(x => x.ID).IsEqualTo(id));
            autoDataModel.LoadModel(null);
            Logger.Send(LogType.Information, "", string.Format("Finished '{0}'", typeof(T).Name));
        }

        private IResponseBuilder HandleTest(IRequest request)
        {
            Logger.Send(LogType.Information, "", "Starting tests");

            var id = Guid.Parse(request.Query["id"]);
            var doc = new Client<Document>().Query(
                new Filter<Document>(x => x.ID).IsEqualTo(id),
                new Columns<Document>(x => x.Data)
            ).Rows[0].ToObject<Document>();

            return request.Respond();
        }

        #region Authentication

        private class LoginSession
        {
            public Guid DatabaseSession { get; set; }

            public User User { get; set; }

            public bool Valid { get; set; }

            public LoginSession(Guid databaseSession, User user, bool valid)
            {
                DatabaseSession = databaseSession;
                User = user;
                Valid = valid;
            }
        }

        private static readonly Dictionary<Guid, LoginSession> Sessions = new();

        private static bool UserCanEdit<T>(User user) where T : Entity, new()
        {
            return Security.CanEdit<T>(user.ID, user.SecurityGroup.ID);
        }

        private LoginSession? GetClientSession(IRequest request)
        {
            if (!request.Cookies.ContainsKey("SessionID")) return null;
            var sessionId = request.Cookies["SessionID"].Value;

            if (Guid.TryParse(sessionId, out var guid))
                return Sessions.GetValueOrDefault(guid);
            return null;
        }

        /// <summary>
        ///     Returns a User object based on a session id cookie.
        /// </summary>
        /// <param name="request"></param>
        /// <returns>null if the user is not logged in, otherwise the current user that is logged in</returns>
        private User? GetClientUser(IRequest request)
        {
            var session = GetClientSession(request);
            if (session?.Valid == true)
                return session.User;
            return null;
        }

        public static Guid LoginUser(User user)
        {
            var newSessionID = Guid.NewGuid();
            Sessions[newSessionID] = new(Guid.Empty, user, true);
            return newSessionID;
        }

        private class ValidateResponse
        {
            public bool Require2FA { get; set; }

            public string? Recipient { get; set; }
        }
        
        private class TwoFARequest
        {
            public string? Code { get; set; }
        }

        private IResponseBuilder Handle2FA(IRequest request)
        {
            var session = GetClientSession(request);
            if(session == null)
            {
                Logger.Send(LogType.Information, "", "Invalid request for 2FA: no Session ID cookie");
                return request.Respond().Status(ResponseStatus.BadRequest);
            }
            else if(request.Content == null)
            {
                Logger.Send(LogType.Information, session.User.UserID, "Invalid request for 2FA: no content");
                return request.Respond().Status(ResponseStatus.BadRequest);
            }

            var requestObj = Serialization.Deserialize<TwoFARequest>(request.Content);
            if (requestObj == null || requestObj.Code == null)
            {
                Logger.Send(LogType.Information, session.User.UserID, "Invalid request for 2FA: invalid content");
                return request.Respond().Status(ResponseStatus.BadRequest);
            }

            Logger.Send(LogType.Information, session.User.UserID, $"2FA check for code {requestObj.Code}");

            if(Client.Check2FA(requestObj.Code, session.DatabaseSession))
            {
                Logger.Send(LogType.Information, session.User.UserID, $"2FA check success!");
                session.Valid = true;
                return request.Respond().Status(ResponseStatus.OK);
            }
            Logger.Send(LogType.Information, session.User.UserID, $"2FA check failed");
            return request.Respond().Status(ResponseStatus.Unauthorized);
        }

        /// <summary>
        ///     www.domain.com/login
        /// </summary>
        /// <param name="request"></param>
        /// <returns></returns>
        private IResponseBuilder HandleLogin(IRequest request)
        {
            var obj = JObject.Parse(new StreamReader(request.Content).ReadToEnd());

            var username = obj["username"]?.ToString();
            var password = obj["password"]?.ToString();
            if (username == null || password == null)
            {
                Logger.Send(LogType.Information, "", "Username or password unspecified");
                return request.Respond().Status(ResponseStatus.BadRequest);
            }
            Logger.Send(LogType.Information, "", string.Format("Request to login with username '{0}'", username));

            var validation = Client.Validate(username, password);
            switch (validation.Status)
            {
                case ValidationStatus.INVALID:
                    Logger.Send(LogType.Information, "", "User name or password is incorrect");
                    return request.Respond().Status(ResponseStatus.Unauthorized);
                case ValidationStatus.PASSWORD_EXPIRED:
                    Logger.Send(LogType.Information, "", "Password has expired");
                    return request.Respond().Status(ResponseStatus.Unauthorized);
            }

            var require2FA = validation.Status == ValidationStatus.REQUIRE_2FA;

            var newSessionID = Guid.NewGuid();
            var user = new User
            {
                ID = validation.UserGuid,
                UserID = validation.UserID,
            };
            user.SecurityGroup.ID = validation.SecurityID;

            Sessions[newSessionID] = new LoginSession(validation.SessionID, user, !require2FA);

            var response = new ValidateResponse
            {
                Require2FA = require2FA
            };
            if (require2FA)
            {
                response.Recipient = validation.Recipient2FA;
                Logger.Send(LogType.Information, "", string.Format("Login ({0}) requires 2FA!", user.UserID));
            }
            else
            {
                Logger.Send(LogType.Information, "", string.Format("Login ({0}) Success!", user.UserID));
            }
            var responseString = Serialization.Serialize(response);

            return request.Respond()
                .Cookie(new Cookie("SessionID", newSessionID.ToString(), LoginExpiry)) // Expires after 30 minutes
                .Content(new ResourceContent(Resource.FromString(responseString).Build()))
                .Type(new FlexibleContentType(ContentType.ApplicationJson))
                .Status(ResponseStatus.OK);
        }

        #endregion

        #region Digital Forms

        private IResponseBuilder HandleFormSubmissionResolvedGeneric<TForm, TEntityLink, TEntity>(IRequest request, User user)
            where TForm : Entity, IDigitalFormInstance<TEntityLink>, IRemotable, IPersistent, new()
            where TEntityLink : IEntityLink<TEntity>
            where TEntity : Entity, IRemotable, IPersistent, new()
        {
            var complete = request.Query.ContainsKey("complete");

            Guid formID;
            if (!request.Query.TryGetValue("id", out var formIDStr))
            {
                Logger.Send(LogType.Error, "", "No form id passed");
                return request.Respond().Status(ResponseStatus.BadRequest);
            } else if(!Guid.TryParse(formIDStr, out formID))
            {
                Logger.Send(LogType.Error, "", "Invalid GUID format '" + formIDStr + "'");
                return request.Respond().Status(ResponseStatus.BadRequest);
            }

            Logger.Send(LogType.Information, user.UserID, $"Request to {(complete ? "submit" : "save")} form '{formID}'");

            if (!Security.CanEdit<TForm>(user.ID, user.SecurityGroup.ID))
            {
                Logger.Send(LogType.Information, user.UserID, string.Format("User does not have the security rights to edit {0}!", typeof(TForm)));
                return request.Respond().Status(ResponseStatus.Forbidden);
            }

            var formTable = new Client<TForm>().Query(
                new Filter<TForm>(x => x.ID).IsEqualTo(formID),
                new Columns<TForm>(x => x.ID).Add(x => x.Form.ID).Add(x => x.Parent.ID)
            );

            if (formTable.Rows.Count == 0)
            {
                Logger.Send(LogType.Error, "", "Form does not exist");
                return request.Respond().Status(ResponseStatus.NotFound);
            }

            var form = formTable.Rows[0].ToObject<TForm>();

            var digitalFormTable = new Client<DigitalForm>().Query(
                new Filter<DigitalForm>(x => x.ID).IsEqualTo(form.Form.ID),
                new Columns<DigitalForm>(x => x.ID)
            );
            if (digitalFormTable.Rows.Count == 0)
            {
                Logger.Send(LogType.Error, "", "Digital form does not exist");
                return request.Respond().Status(ResponseStatus.NotFound);
            }

            var digitalForm = digitalFormTable.Rows[0].ToObject<DigitalForm>();

            var json = new StreamReader(request.Content).ReadToEnd();
            var values = Serialization.Deserialize<Dictionary<string, object>>(json);

            var variables = new Client<DigitalFormVariable>().Query(
                new Filter<DigitalFormVariable>(x => x.Form.ID).IsEqualTo(digitalForm.ID),
                new Columns<DigitalFormVariable>(x => x.Code).Add(x => x.VariableType).Add(x => x.Parameters)
            ).Rows.Select(x => x.ToObject<DigitalFormVariable>()).ToArray();

            var entity = new Client<TEntity>().Query(
                new Filter<TEntity>(x => x.ID).IsEqualTo(form.Parent.ID)
            ).Rows.FirstOrDefault()?.ToObject<TEntity>();

            var formData = DigitalForm.GenerateFormData(values, variables, entity);
            if (formData == null)
            {
                Logger.Send(LogType.Error, "", "Not all required fields are present");
                return request.Respond().Status(ResponseStatus.BadRequest);
            }

            form.FormData = formData;

            if (complete)
            {
                form.FormCompleted = DateTime.Now;
                form.FormCompletedBy = new UserLink { ID = user.ID };
            }

            new Client<TForm>().Save(form, $"{(complete ? "Submitted" : "Saved")} by '{user.UserID}' from the web");
            if (entity?.IsChanged() == true) new Client<TEntity>().Save(entity, $"Saved by '{user.UserID}' from the web");

            Logger.Send(LogType.Information, user.UserID, $"Form '{formID}' {(complete ? "submitted" : "saved")}.");

            return request.Respond().Status(ResponseStatus.OK);
        }

        /// <summary>
        ///     For urls of the form 'www.domain.com/form_submission/XXXXForm?id=XXXX'
        /// </summary>
        /// <param name="request"></param>
        /// <returns></returns>
        private IResponseBuilder HandleFormSubmission(IRequest request)
        {
            var user = GetClientUser(request);
            if (user == null)
            {
                Logger.Send(LogType.Information, "", "User is not logged in!");
                return request.Respond().Status(ResponseStatus.Forbidden);
            }

            if (request.Content == null)
                return request.Respond().Status(ResponseStatus.BadRequest);
            if (request.ContentType.KnownType != ContentType.ApplicationJson) return request.Respond().Status(ResponseStatus.UnsupportedMediaType);

            // Get EntityForm Type
            var remaining = request.Target.GetRemaining();
            if (remaining.Parts.Count < 2) return request.Respond().Status(ResponseStatus.NotFound);
            var formTypeName = remaining.Parts[1].Value;
            var formType = GetPersistentRemotableEntities(formTypeName).FirstOrDefault();
            if (formType == null || !formType.IsSubclassOfRawGeneric(typeof(EntityForm<,,>))) return request.Respond().Status(ResponseStatus.NotFound);

            var entityTypes = formType.GetSuperclassDefinition(typeof(EntityForm<,,>)).GetGenericArguments();

            var genericMethod = typeof(WebHandler).GetMethod(nameof(HandleFormSubmissionResolvedGeneric), BindingFlags.Instance | BindingFlags.NonPublic);
            return genericMethod.MakeGenericMethod(formType, entityTypes[1], entityTypes[0]).Invoke(this, new object[] { request, user }) as
                IResponseBuilder;
        }

        #endregion

        #region RazorEngine

        /// <summary>
        ///     Associates a "template.DataModel/template.Slug" to (template source, template Key)
        /// </summary>
        private static readonly Dictionary<string, Tuple<string, string>> templates = new();

        private static int Key;

        private static IRazorEngineService RazorService { get; set; }

        private void CompileAllTemplates()
        {
            var templates = new Client<WebTemplate>().Query(
                null,
                new Columns<WebTemplate>().Add(x => x.DataModel).Add(x => x.Slug).Add(x => x.Template)
            );
            foreach (var templateRow in templates.Rows)
            {
                var template = templateRow.ToObject<WebTemplate>();
                try
                {
                    Logger.Send(LogType.Information, "",
                        string.Format("Compiling Template {0}{1}", !string.IsNullOrWhiteSpace(template.DataModel) ? template.DataModel + "/" : "",
                            template.Slug));
                    CompileTemplate(template);
                }
                catch (Exception e)
                {
                    LogTemplateError(e);
                }
            }
        }

        private static void LogTemplateError(Exception e)
        {
            var templateStart = e.Message.IndexOf("The template we tried to compile is: "); // Chop from here
            if (templateStart == -1) templateStart = e.Message.IndexOf("------------- START -----------"); // If that doesn't exist, chop from here
            if (templateStart == -1) templateStart = e.Message.Length;

            Logger.Send(LogType.Error, "", string.Format("Error while compiling or running template: {0}", e.Message.Substring(0, templateStart)));
        }

        public static void CompileTemplate(WebTemplate template)
        {
            var templateKey = template.DataModel + "/" + template.Slug;
            if (templates.TryGetValue(templateKey, out var cachedTemplate))
            {
                if (template.Template != cachedTemplate.Item1)
                {
                    var newKey = (Key++).ToString();
                    RazorService.Compile(template.Template, newKey, typeof(WebModel));
                    templates[templateKey] = new Tuple<string, string>(template.Template, newKey);
                }
            }
            else
            {
                var newKey = (Key++).ToString();
                RazorService.Compile(template.Template, newKey, typeof(WebModel));
                templates[templateKey] = new Tuple<string, string>(template.Template, newKey);
            }
        }

        public static string? RunTemplate(WebTemplate template, object? model = null)
        {
            try
            {
                var templateKey = template.DataModel + "/" + template.Slug;
                if (templates.ContainsKey(templateKey))
                {
                    var cachedTemplate = templates[templateKey];
                    if (template.Template != cachedTemplate.Item1)
                    {
                        var newKey = (Key++).ToString();
                        var result = RazorService.RunCompile(template.Template, newKey, typeof(WebModel), model);
                        templates[templateKey] = new Tuple<string, string>(template.Template, newKey);
                        return result;
                    }
                    else
                    {
                        var result = RazorService.Run(cachedTemplate.Item2, typeof(WebModel), model);
                        return result;
                    }
                }

                {
                    var newKey = (Key++).ToString();
                    var result = RazorService.RunCompile(template.Template, newKey, typeof(WebModel), model);
                    templates[templateKey] = new Tuple<string, string>(template.Template, newKey);
                    return result;
                }
            }
            catch (Exception e)
            {
                LogTemplateError(e);
                return null;
            }
        }

        private WebTemplate GenerateNewPageForDataModel(string dataModel)
        {
            var html = @"@using PRSServer;
@using Comal.Classes;
@using InABox.Core;
@using InABox.Clients;
@using System.Reflection;
@using System.Text;
@{
Model.LoadModel(
    new string[] { ""CompanyInformation"", ""CompanyLogo"", """ + dataModel + @""", ""User"" }
);
}
<!DOCTYPE html>
<html>
    <head>
        <title>" + dataModel + @"</title>
        <meta charset=""utf8"">
        <meta name=""viewport"" content=""width = device-width,initial-scale = 1"">
        <style>
        @(WebDatabaseInterface.GetStylesheet(""DEFAULT"").Style)
        </style>
    </head>
    <body>
        <header class=""prs-web_engine-header"">
    		@{CoreTable companyInformationTable = Model.AsDictionary[typeof(CompanyInformation)];
    		if(companyInformationTable.Rows.Count > 0){
    			CompanyInformation companyInformation = companyInformationTable.Rows[0].ToObject<CompanyInformation>();
    			
    			CoreTable documentTable = Model.AsDictionary[typeof(Document)];
    			Document logo = null;
    			foreach(var row in documentTable.Rows){
    				if((Guid)row[""ID""] == companyInformation.Logo.ID){
    					logo = row.ToObject<Document>();
    				}
    			}
                <div class=""left"">
        			@if(logo != null){
        				<img src=""data:;base64,@(Convert.ToBase64String(logo.Data))"" class=""prs-web_engine-header_img"" />
        			}
        		</div>
    			<div class=""center"">
    			    <h1>@companyInformation.CompanyName</h1>
    			</div>
                <div class=""right"">
                    <ul class=""prs-web_engine-header_address"">
                        <li>@(companyInformation.PostalAddress.Street)</li>
                        <li>@(companyInformation.PostalAddress.City), @(companyInformation.PostalAddress.State) @(companyInformation.PostalAddress.PostCode)</li>
                    </ul>
                </div>
    		}
        </header>
        <main>
            <h1>@Model.Name</h1>
            @if(WebUtils.UserCanAccess<AutoSecurityDescriptor<" + dataModel + @",CanView<" + dataModel + @">>>(Model)){
                <div class=""table_scroll"">
                    @Raw(WebUtils.BuildHTMLTableFromCoreTable<" + dataModel + @">(Model.AsDictionary[typeof(" + dataModel +
                       @")],WebUtils.EntityTableColumns.VISIBLE))
                </div>
                foreach(var item in Model.AsDictionary){
                    if(item.Key != typeof(" + dataModel + @") && Model.IsChildTable(item.Key.Name) && item.Value.Rows.Count > 0){
                        @:<br/>
                        <div class=""table_scroll"">
                            @Raw(WebUtils.BuildHTMLTableFromCoreTable(item.Key, item.Value, WebUtils.EntityTableColumns.VISIBLE))
                        </div>
                    }
                }
            }
        </main>
    </body>
</html>";
            var template = new WebTemplate();
            template.Template = html;

            return template;
        }

        private List<Type>? _persistentRemotable;

        private IEnumerable<Type> GetPersistentRemotableEntities(string entityName)
        {
            _persistentRemotable ??= CoreUtils.TypeList(
                e => e.GetInterfaces().Contains(typeof(IRemotable)) &&
                     e.GetInterfaces().Contains(typeof(IPersistent))).ToList();
            return _persistentRemotable.Where(x => x.Name == entityName);
        }

        private void PrepareDataModel(DataModel model, IRequest request, User? user)
        {
            var userTable = new CoreTable();
            userTable.LoadColumns(typeof(User));
            if (user != null) 
                userTable.LoadRow(user);
            model.SetTableData(typeof(User), userTable);

            var requestQueryTable = new CoreTable();
            requestQueryTable.Columns.Add(new CoreColumn { ColumnName = "Key", DataType = typeof(string) });
            requestQueryTable.Columns.Add(new CoreColumn { ColumnName = "Value", DataType = typeof(string) });
            foreach (var query in request.Query)
            {
                var row = requestQueryTable.NewRow();
                row["Key"] = query.Key;
                row["Value"] = query.Value;
                requestQueryTable.Rows.Add(row);
            }

            model.AddTable(typeof(CoreTable), requestQueryTable, alias: "Queries");
        }

        private IResponseBuilder GetResponse(WebTemplate template, WebModel model)
        {
            var compiledHTML = RunTemplate(template, model) ?? "";

            if (model.Response != null) return model.Response;

            var response = model.Request.Respond()
                .Content(compiledHTML)
                .Type(new FlexibleContentType(ContentType.TextHtml));
            foreach (var (key, value) in model.Cookies)
            {
                response.Cookie(new Cookie(key, value));
            }
            return response;
        }

        /// <summary>
        ///     Literally exists so that minimising reflection code. Called once entityType has been resolved
        /// </summary>
        /// <typeparam name="T"></typeparam>
        /// <returns></returns>
        private IResponseBuilder HandleDataModelWithResolvedEntity<T>(IRequest request, string slug, string dataModelName,
            Guid entityId, bool isRoot) where T : Entity, IPersistent, IRemotable, new()
        {
            var templates = new Client<WebTemplate>().Query(
                new Filter<WebTemplate>(x => x.Slug).IsEqualTo(slug)
                    .And(x => x.DataModel).IsEqualTo(dataModelName)
                    .And(x => x.RootURL).IsEqualTo(isRoot),
                new Columns<WebTemplate>(x => x.Template).Add(x => x.Visible)).Rows;

            WebTemplate template;
            if (templates.Count == 0)
            {
                template = GenerateNewPageForDataModel(dataModelName);
                Logger.Send(LogType.Information, "ADMIN", "Template generated");
            }
            else
            {
                template = templates[0].ToObject<WebTemplate>();
                if (!template.Visible)
                {
                    Logger.Send(LogType.Information, "ADMIN", "Template is not public");
                    return request.Respond().Status(ResponseStatus.NotFound);
                }

                Logger.Send(LogType.Information, "ADMIN", "Template retrieved");
            }

            var dataModel = new AutoDataModel<T>(new Filter<T>(x => x.ID).IsEqualTo(entityId));

            var user = GetClientUser(request);
            PrepareDataModel(dataModel, request, user);

            var model = new WebModel(dataModel, request, user);
            return GetResponse(template, model);
        }

        // For requests like 'www.domain.com/v1/slug'
        private IResponseBuilder HandleWebTemplate(IRequest request, string slug, bool isRoot = false)
        {
            var templates = new Client<WebTemplate>().Query(
                new Filter<WebTemplate>(x => x.Slug).IsEqualTo(slug)
                    .And(x => x.DataModel).IsEqualTo("")
                    .And(x => x.RootURL).IsEqualTo(isRoot),
                new Columns<WebTemplate>(x => x.Template).Add(x => x.Visible)).Rows;
            WebTemplate template;
            if (templates.Count == 0)
            {
                Logger.Send(LogType.Information, "ADMIN", "Template does not exist");
                return request.Respond().Status(ResponseStatus.NotFound);
            }

            template = templates[0].ToObject<WebTemplate>();
            if (!template.Visible)
            {
                Logger.Send(LogType.Information, "ADMIN", "Template is not public");
                return request.Respond().Status(ResponseStatus.NotFound);
            }

            Logger.Send(LogType.Information, "ADMIN", "Template retrieved");

            var dataModel = new EmptyDataModel();
            var user = GetClientUser(request);
            PrepareDataModel(dataModel, request, user);
            
            var model = new WebModel(dataModel, request, user);
            return GetResponse(template, model);
        }

        // For requests like 'www.domain.com/v1/*'
        private IResponseBuilder HandleDataModel(IRequest request, IList<string> endpoint, bool isRoot = false)
        {
            if (endpoint.Count == 0)
                return HandleUnknown(request);
            if (endpoint.Count == 1) return HandleWebTemplate(request, endpoint[0], isRoot);

            // Matches the DataModel entry in WebTemplate
            var dataModelName = endpoint[0];

            // Matches the Slug entry in WebTemplate
            var slug = endpoint[1];

            var entityId = request.Query.GetValueOrDefault("id");
            if (entityId == null)
            {
                Logger.Send(LogType.Error, "", "No entity id passed");
                return request.Respond().Status(ResponseStatus.BadRequest);
            }

            if(!Guid.TryParse(entityId, out var guid))
            {
                Logger.Send(LogType.Error, "", "Invalid GUID format '" + entityId + "'");
                return request.Respond().Status(ResponseStatus.BadRequest);
            }

            Logger.Send(LogType.Information, "ADMIN",
                "Request to Datamodel '" + dataModelName + "' with slug '" + slug + "' and entity id '" + entityId + "'");

            var entityTypes = GetPersistentRemotableEntities(dataModelName);
            if (entityTypes.Count() == 0)
            {
                Logger.Send(LogType.Error, "", "No entity named '" + dataModelName + "'");
                return request.Respond().Status(ResponseStatus.NotFound);
            }

            var entityType = entityTypes.First();

            var method = typeof(WebHandler).GetMethods(BindingFlags.NonPublic | BindingFlags.Instance)
                .Where(x => x.Name == nameof(HandleDataModelWithResolvedEntity))
                .FirstOrDefault()!.MakeGenericMethod(entityType);

            return (method.Invoke(this, new object[] { request, slug, dataModelName, guid, isRoot }) as IResponseBuilder)!;
        }

        #endregion

        #region Edit

        public static Dictionary<string, object?> SerializeEntityForEditing(Type entityType, Entity entity)
        {
            var data = new Dictionary<string, object?>();
            foreach (var property in DatabaseSchema.Properties(entityType))
            {
                var editor = EditorUtils.GetPropertyEditor(entityType, property);
                if (editor != null && !(editor is NullEditor) && editor.Editable.EditorVisible())
                {
                    if (editor is LookupEditor)
                    {
                        data[property.Name + "$ID"] = CoreUtils.GetPropertyValue(entity, property.Name);
                    }
                    else
                    {
                        data[property.Name] = CoreUtils.GetPropertyValue(entity, property.Name);
                    }
                }
            }

            data["ID"] = entity.ID;
            return data;
        }

        public static T DeserializeEditedEntity<T>(Dictionary<string, object> data) where T : Entity, IRemotable, IPersistent, new()
        {
            var id = data.GetValueOrDefault("ID");
            T entity;
            if (id != null && id is string idStr && Guid.TryParse(idStr, out var guid) && guid != Guid.Empty)
                entity = new Client<T>().Load(new Filter<T>(x => x.ID).IsEqualTo(guid)).FirstOrDefault() ?? new T();
            else
                entity = new T();

            foreach (var property in DatabaseSchema.Properties(typeof(T)))
            {
                var editor = EditorUtils.GetPropertyEditor(typeof(T), property);

                var value = editor is LookupEditor ?
                    (data.GetValueOrDefault(property.Name + "$ID") ?? data.GetValueOrDefault(property.Name)) :
                    data.GetValueOrDefault(property.Name);
                if (value != null)
                {
                    CoreUtils.SetPropertyValue(entity, property.Name, CoreUtils.ChangeType(value, property.PropertyType));
                }
            }
            return entity;
        }

        /// <summary>
        ///     www.domain.com/save/XXXX
        ///     A POST request to this URL with JSON data representing an entity saves the entity to the database.
        /// </summary>
        private IResponseBuilder HandleSave(IRequest request)
        {
            var endpoint = request.Target.GetRemaining();
            Logger.Send(LogType.Information, "ADMIN", "Request to endpoint '" + endpoint + "'");

            if (endpoint.Parts.Count < 2)
                return request.Respond().Status(ResponseStatus.NotFound);
            if (request.Content == null)
                return request.Respond().Status(ResponseStatus.BadRequest);
            if (request.ContentType?.KnownType != ContentType.ApplicationJson) return request.Respond().Status(ResponseStatus.UnsupportedMediaType);

            var user = GetClientUser(request);
            if (user == null)
            {
                Logger.Send(LogType.Information, "", "User is not logged in!");
                return request.Respond().Status(ResponseStatus.Forbidden);
            }

            var entityName = endpoint.Parts[1].Value;
            var entityType = GetPersistentRemotableEntities(entityName).FirstOrDefault();
            if (entityType == null)
            {
                Logger.Send(LogType.Information, "", string.Format("Entity {0} does not exist", entityName));
                return request.Respond().Status(ResponseStatus.NotFound);
            }

            var securityMethod = typeof(WebHandler).GetMethods(BindingFlags.NonPublic | BindingFlags.Static)
                .Where(x => x.Name == "UserCanEdit").FirstOrDefault()?.MakeGenericMethod(entityType);
            if ((bool?)securityMethod?.Invoke(this, new object[] { user }) != true)
            {
                Logger.Send(LogType.Information, "", string.Format("User does not have the security rights to edit {0}!", entityName));
                return request.Respond().Status(ResponseStatus.Forbidden);
            }

            var dictionary = Serialization.Deserialize<Dictionary<string, object>>(request.Content);
            var entity = (typeof(WebHandler).GetMethod(nameof(WebHandler.DeserializeEditedEntity), BindingFlags.Static | BindingFlags.Public)!
                .MakeGenericMethod(entityType).Invoke(this, new object?[] { dictionary }) as Entity)!;

            var client = ClientFactory.CreateClient(entityType);
            client.Save(entity, string.Format("Edited by {0} from the web", user.UserID));
            Logger.Send(LogType.Information, user.UserID, string.Format("{0} saved with ID '{1}'", entityName, entity.ID));

            var returnJSON = Serialization.Serialize(SerializeEntityForEditing(entityType, entity));

            return request.Respond()
                .Content(new ResourceContent(Resource.FromString(returnJSON).Build()))
                .Type(FlexibleContentType.Get(ContentType.ApplicationJson))
                .Status(ResponseStatus.OK);
        }

        #endregion

        #region Documents

        private class DocumentData
        {
            public DocumentData(byte[] data, string fileName, bool priv)
            {
                Data = data;
                FileName = fileName;
                Private = priv;
            }

            public byte[] Data { get; }

            public string FileName { get; }

            public bool Private { get; }
        }

        private enum DocumentError
        {
            NOT_FOUND,
            TOO_LARGE
        }

        private DocumentData? GetDocumentByID(Guid guid)
        {
            var documentTable = new Client<Document>().Query(
                new Filter<Document>(x => x.ID).IsEqualTo(guid),
                new Columns<Document>(x => x.Data).Add(x => x.FileName).Add(x => x.Private)
            );
            if (documentTable.Rows.Count == 0) return null;
            var row = documentTable.Rows[0];
            return new DocumentData(
                row.Get<Document, byte[]>(x => x.Data),
                row.Get<Document, string>(x => x.FileName),
                row.Get<Document, bool>(x => x.Private));
        }

        private IResponseBuilder HandleDocument(IRequest request)
        {
            var user = GetClientUser(request);
            var canViewPrivate = user != null && Security.CanView<Document>(user.ID, user.SecurityGroup.ID);

            var documentId = request.Query.GetValueOrDefault("id");

            DocumentData? document;

            if (documentId != null)
            {
                if(!Guid.TryParse(documentId, out var guid))
                {
                    Logger.Send(LogType.Error, "", "Invalid GUID format '" + documentId + "'");
                    return request.Respond().Status(ResponseStatus.BadRequest);
                }

                Logger.Send(LogType.Error, "", string.Format("Request to document with id '{0}'", documentId));

                document = GetDocumentByID(guid);
            }
            else
            {
                Logger.Send(LogType.Error, "", "No document id passed");
                return request.Respond().Status(ResponseStatus.BadRequest);
            }

            if (document == null) return request.Respond().Status(ResponseStatus.NotFound);
            if (document.Private && !canViewPrivate)
            {
                Logger.Send(LogType.Error, "", "User is not authorised to access this resource");
                return request.Respond().Status(ResponseStatus.Forbidden);
            }

            var extension = Path.GetExtension(document.FileName).ToLower();

            if (request.Query.GetValueOrDefault("to_img") != null && extension == ".pdf")
            {
                var imgs = WebUtils.RenderPDFToImages(document.Data);

                var responseString = Serialization.Serialize(imgs);

                return request.Respond()
                    .Content(new ResourceContent(Resource.FromString(responseString).Build()))
                    .Type(new FlexibleContentType(ContentType.ImageJpg));
            }

            var contentType = extension switch
            {
                ".pdf" => ContentType.TextPlain,
                ".png" => ContentType.ImagePng,
                _ => ContentType.TextPlain
            };
            return request.Respond()
                .Content(new ByteArrayResource(document.Data, document.FileName, FlexibleContentType.Get(contentType), null))
                .Type(new FlexibleContentType(contentType));
        }

        #endregion

        #region Root Handlers

        private IResponseBuilder HandleGet(IRequest request)
        {
            var endpoint = request.Target.GetRemaining();
            if (endpoint.Parts.Count == 0)
            {
                Logger.Send(LogType.Error, "", $"Request to endpoint '{endpoint}' from IP {request.Client.IPAddress} does not exist!");
                return request.Respond().Status(ResponseStatus.NotFound);
            }

            Logger.Send(LogType.Information, "ADMIN", "Request to endpoint '" + endpoint + "'");

            var endpointNamespace = endpoint.Parts[0].Value;
            var responseBuilder = endpointNamespace switch
            {
                "v1" => HandleDataModel(request, endpoint.Parts.Skip(1).Select(x => x.Value).ToList(), false),
                "document" => HandleDocument(request),
                "test" => HandleTest(request),
                _ => HandleDataModel(request, endpoint.Parts.Select(x => x.Value).ToList(), true)
            };
            if(responseBuilder == null)
            {
                Logger.Send(LogType.Error, "", $"Request to endpoint '{endpoint}' from IP {request.Client.IPAddress} does not exist!");
                return request.Respond().Status(ResponseStatus.NotFound);
            }

            if (request.Cookies.TryGetValue("SessionID", out var sessionID))
                try
                {
                    var guid = Guid.Parse(sessionID.Value);
                    if (Sessions.ContainsKey(guid)) responseBuilder.Cookie(new Cookie("SessionID", sessionID.Value, LoginExpiry));
                }
                catch (FormatException)
                {
                }

            return responseBuilder;
        }

        private IResponseBuilder HandleUnknown(IRequest request)
        {
            Logger.Send(LogType.Error, "", $"{request.Method.RawMethod} request to endpoint '{request.Target.Path}' from IP {request.Client.IPAddress} does not exist!");
            return request.Respond().Status(ResponseStatus.NotFound);
        }

        public override ValueTask<IResponse?> HandleAsync(IRequest request)
        {
            try
            {
                return request.Method.KnownMethod switch
                {
                    RequestMethod.GET or RequestMethod.HEAD => new ValueTask<IResponse?>(HandleGet(request).Build()),
                    RequestMethod.POST => new ValueTask<IResponse?>((request.Target.Current?.Value switch
                    {
                        "login" => HandleLogin(request),
                        "2fa" => Handle2FA(request),
                        "save" => HandleSave(request),
                        "form_submission" => HandleFormSubmission(request),
                        "v1" => HandleDataModel(request, request.Target.GetRemaining().Parts.Skip(1).Select(x => x.Value).ToList(), false),
                        _ => HandleDataModel(request, request.Target.GetRemaining().Parts.Select(x => x.Value).ToList(), true)
                    }).Build()),
                    _ => new ValueTask<IResponse?>(request.Respond().Status(ResponseStatus.MethodNotAllowed).Header("Allow", "GET, HEAD").Build())
                };
            }
            catch (Exception eListen)
            {
                Logger.Send(LogType.Error, ClientFactory.UserID, eListen.Message);
                return new ValueTask<IResponse?>(request.Respond().Status(ResponseStatus.InternalServerError).Build());
            }
        }

        #endregion
    }
    
    public class WebHandlerProperties
    {
        public int MaxFileSize { get; }

        public ulong LoginExpiry { get; }

        /// <summary>
        /// Create a new web listener
        /// </summary>
        /// <param name="maxFileSize">The maximum size in megabytes for file transfer</param>
        /// <param name="loginExpiry">The time, in seconds, for how long login sessions should last</param>
        public WebHandlerProperties(int maxFileSize, ulong loginExpiry)
        {
            MaxFileSize = maxFileSize;
            LoginExpiry = loginExpiry;
        }
    }

    public class WebListener : Listener<WebHandler, WebHandlerProperties>
    {
        public WebListener(WebHandlerProperties properties): base(properties) { }
    }
}