diff --git a/Library.Encyclopedia.API/Controllers/EncylopediaController.cs b/Library.Encyclopedia.API/Controllers/EncylopediaController.cs index 023d81a..ca21e77 100644 --- a/Library.Encyclopedia.API/Controllers/EncylopediaController.cs +++ b/Library.Encyclopedia.API/Controllers/EncylopediaController.cs @@ -1,5 +1,6 @@ using Library.Encyclopedia.DataAccess; using Library.Encyclopedia.DataAccess.DataAccess; +using Library.Encyclopedia.DataAccess.QueryStatsAccess; using Library.Encyclopedia.Entity.Interfaces; using Library.Encyclopedia.Entity.Models; using Library.Encyclopedia.Entity.Models.External; diff --git a/Library.Encyclopedia.API/Controllers/StatisticsController.cs b/Library.Encyclopedia.API/Controllers/StatisticsController.cs new file mode 100644 index 0000000..8d5c0e9 --- /dev/null +++ b/Library.Encyclopedia.API/Controllers/StatisticsController.cs @@ -0,0 +1,68 @@ +using Library.Encyclopedia.DataAccess; +using Library.Encyclopedia.DataAccess.QueryStatsAccess; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Caching.Memory; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Library.Encyclopedia.API.Controllers +{ + [ApiController] + [Route("[controller]")] + public class StatisticsController : ControllerBase + { + private readonly ILogger _logger; + private readonly CommonlyOccuringWordsAdapter commonlyOccuringWordsAdapter; + + public StatisticsController(ILogger logger, IApplicationDbContext applicationDbContext, IMemoryCache memoryCache) + { + _logger = logger; + this.commonlyOccuringWordsAdapter = new CommonlyOccuringWordsAdapter(applicationDbContext, memoryCache); + } + + [HttpGet("GetRecommendedWords")] + public IActionResult GetCommonlyOccuringWordsStats(int count = 10) + { + try + { + return Ok(commonlyOccuringWordsAdapter.GetCommonlyUsedWords(count)); + } + catch (Exception ex) + { + _logger.LogError(ex, $"an error has occured {ex.Message}"); + throw; + } + } + + [HttpGet("GetPopularWords")] + public IActionResult GetMostSearchedWordStats(int count = 10) + { + try + { + return Ok(commonlyOccuringWordsAdapter.GetCommonlySearchedWords(count)); + } + catch (Exception ex) + { + _logger.LogError(ex, $"an error has occured {ex.Message}"); + throw; + } + } + + [HttpGet("GetNLPResult")] + public IActionResult GetNLPResult() + { + try + { + return Ok(commonlyOccuringWordsAdapter.GetNLPResult()); + } + catch (Exception ex) + { + _logger.LogError(ex, $"an error has occured {ex.Message}"); + throw; + } + } + } +} diff --git a/Library.Encyclopedia.API/Startup.cs b/Library.Encyclopedia.API/Startup.cs index a87594b..96f30d8 100644 --- a/Library.Encyclopedia.API/Startup.cs +++ b/Library.Encyclopedia.API/Startup.cs @@ -1,5 +1,6 @@ using Library.Encyclopedia.DataAccess; using Library.Encyclopedia.DataAccess.FileAccess; +using Library.Encyclopedia.DataAccess.QueryStatsAccess; using Library.Encyclopedia.Entity.Interfaces; using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Hosting; @@ -64,7 +65,8 @@ public void ConfigureServices(IServiceCollection services) }); } - services.AddScoped(s => new ApplicationDbContext(Configuration.GetConnectionString("DefaultConnection"))); + services.AddSingleton(s => new ApplicationDbContext(Configuration.GetConnectionString("DefaultConnection"))); + services.AddMemoryCache(); services.Configure(x => { diff --git a/Library.Encyclopedia.DataAccess/ApplicationDbContext.cs b/Library.Encyclopedia.DataAccess/ApplicationDbContext.cs index 574dc9d..0c6069e 100644 --- a/Library.Encyclopedia.DataAccess/ApplicationDbContext.cs +++ b/Library.Encyclopedia.DataAccess/ApplicationDbContext.cs @@ -1,5 +1,6 @@ using Library.Encyclopedia.Entity.Models; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; using System; using System.Threading.Tasks; @@ -16,16 +17,24 @@ public ApplicationDbContext(string connectionString) protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder) { var serverVersion = new MySqlServerVersion(new Version(8, 0, 18)); - optionsBuilder.UseMySql(connectionString, serverVersion); + optionsBuilder.UseMySql(connectionString, serverVersion, (mySqlOptions)=> { + mySqlOptions.EnableRetryOnFailure(); + }); } public DbSet
Main { get; set; } public DbSet Files { get; set; } public DbSet Links { get; set; } + public DbSet QueryStats { get; set; } public new async Task SaveChanges() { return await base.SaveChangesAsync(); } + + public async Task BeginTransactionAsync() + { + return await base.Database.BeginTransactionAsync(); + } } } \ No newline at end of file diff --git a/Library.Encyclopedia.DataAccess/DataAccess/MainDataAccess.cs b/Library.Encyclopedia.DataAccess/DataAccess/MainDataAccess.cs index 64895af..95ccfe9 100644 --- a/Library.Encyclopedia.DataAccess/DataAccess/MainDataAccess.cs +++ b/Library.Encyclopedia.DataAccess/DataAccess/MainDataAccess.cs @@ -1,4 +1,5 @@ -using Library.Encyclopedia.Entity.Exceptions; +using Library.Encyclopedia.DataAccess.QueryStatsAccess; +using Library.Encyclopedia.Entity.Exceptions; using Library.Encyclopedia.Entity.Interfaces; using Library.Encyclopedia.Entity.Models; using Library.Encyclopedia.Entity.Models.External; @@ -10,6 +11,7 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; +using System.Threading; using System.Threading.Tasks; using System.Web; @@ -19,6 +21,7 @@ public class MainDataAccess : IMainDataAccess { private IApplicationDbContext _dbcontext; private readonly IFilesAdapter filesAdapter; + private readonly QueryStatsAdapter queryStatsAdapter; private string APP_BASE_URL; public MainDataAccess(IApplicationDbContext dbcontext, IConfiguration configuration, IFilesAdapter filesAdapter) @@ -26,6 +29,7 @@ public MainDataAccess(IApplicationDbContext dbcontext, IConfiguration configurat APP_BASE_URL = configuration.GetSection("App-Base-Url").Value; _dbcontext = dbcontext; this.filesAdapter = filesAdapter; + this.queryStatsAdapter = new QueryStatsAdapter(dbcontext); } #region GET @@ -55,6 +59,9 @@ async Task IMainDataAccess.GetAsync(string quer MainMinimizedExternalCollection result = new MainMinimizedExternalCollection(data.MinimizeWithQuery(query, previewSize), total); + if (!string.IsNullOrEmpty(query) && !string.IsNullOrWhiteSpace(query) && total > 0 && query.Length >= 3) + this.queryStatsAdapter.AddQuery(query); + return result; } @@ -100,7 +107,7 @@ async Task IMainDataAccess.GetByCategoryAsync(s .ToListAsync(); else data = await _dbcontext.Main.Where(s => s.Category == null) - + .OrderByDescending(s => s.Title.ToLower()) .ThenByDescending(s => s.RawDescription.ToLower()) .Skip(offset) diff --git a/Library.Encyclopedia.DataAccess/IApplicationDbContext.cs b/Library.Encyclopedia.DataAccess/IApplicationDbContext.cs index 4c1fcf0..fb924c7 100644 --- a/Library.Encyclopedia.DataAccess/IApplicationDbContext.cs +++ b/Library.Encyclopedia.DataAccess/IApplicationDbContext.cs @@ -1,5 +1,6 @@ using Library.Encyclopedia.Entity.Models; using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage; using System.Threading.Tasks; namespace Library.Encyclopedia.DataAccess @@ -9,6 +10,8 @@ public interface IApplicationDbContext DbSet
Main { get; set; } DbSet Files { get; set; } DbSet Links { get; set; } + DbSet QueryStats { get; set; } Task SaveChanges(); + Task BeginTransactionAsync(); } } \ No newline at end of file diff --git a/Library.Encyclopedia.DataAccess/Library.Encyclopedia.DataAccess.csproj b/Library.Encyclopedia.DataAccess/Library.Encyclopedia.DataAccess.csproj index e53781f..c75a2ba 100644 --- a/Library.Encyclopedia.DataAccess/Library.Encyclopedia.DataAccess.csproj +++ b/Library.Encyclopedia.DataAccess/Library.Encyclopedia.DataAccess.csproj @@ -5,6 +5,8 @@ + + @@ -18,6 +20,7 @@ + diff --git a/Library.Encyclopedia.DataAccess/QueryStatsAccess/CommonlyOccuringWordsAdapter.cs b/Library.Encyclopedia.DataAccess/QueryStatsAccess/CommonlyOccuringWordsAdapter.cs new file mode 100644 index 0000000..681bd64 --- /dev/null +++ b/Library.Encyclopedia.DataAccess/QueryStatsAccess/CommonlyOccuringWordsAdapter.cs @@ -0,0 +1,175 @@ +using Catalyst; +using Catalyst.Models; +using Library.Encyclopedia.Entity.Models; +using Microsoft.Extensions.Caching.Memory; +using Mosaik.Core; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +namespace Library.Encyclopedia.DataAccess.QueryStatsAccess +{ + public class CommonlyOccuringWordsAdapter + { + private readonly IApplicationDbContext applicationDbContext; + private Dictionary commonlyOccuringWords; + private Dictionary> forReference; + private IMemoryCache _cache; + + public CommonlyOccuringWordsAdapter(IApplicationDbContext applicationDbContext, IMemoryCache cache) + { + commonlyOccuringWords = new Dictionary(); + forReference = new Dictionary>(); + this.applicationDbContext = applicationDbContext; + this._cache = cache; + } + + private void PopulateRecommendedWords() + { + commonlyOccuringWords = new Dictionary(); + forReference = new Dictionary>(); + + var allData = applicationDbContext.Main.ToList(); + + //BasicParser(allData); + AdvancedParser(allData); + + var cacheEntryOptions = new MemoryCacheEntryOptions() + .SetSlidingExpiration(TimeSpan.FromHours(1)); + + if (commonlyOccuringWords != null) + { + _cache.Set("commonlyOccuringWords", commonlyOccuringWords, cacheEntryOptions); + } + } + + private void AdvancedParser(List
allData) + { + English.Register(); + Storage.Current = new DiskStorage("catalyst-models"); + var nlp = Pipeline.For(Language.English); + + foreach (var item in allData) + { + ProcessText(nlp, item.Title, item.Id); + ProcessText(nlp, item.RawDescription, item.Id); + } + } + + private void ProcessText(Pipeline nlp, string text, Guid id) + { + var doc = new Document(text, Language.English); + nlp.ProcessSingle(doc); + + foreach (var sentence in doc) + { + foreach (var token in sentence) + { + switch (token.POS) + { + case PartOfSpeech.PROPN: + case PartOfSpeech.NOUN: + case PartOfSpeech.ADJ: + case PartOfSpeech.ADV: + case PartOfSpeech.NUM: + if (token.Value.Length >= 3) + { + if (commonlyOccuringWords.ContainsKey(token.Value)) + commonlyOccuringWords[token.Value]++; + else + commonlyOccuringWords.Add(token.Value, 1); + } + break; + case PartOfSpeech.NONE: + case PartOfSpeech.ADP: + case PartOfSpeech.AUX: + case PartOfSpeech.CCONJ: + case PartOfSpeech.DET: + case PartOfSpeech.INTJ: + case PartOfSpeech.PART: + case PartOfSpeech.PRON: + case PartOfSpeech.PUNCT: + case PartOfSpeech.SCONJ: + case PartOfSpeech.SYM: + case PartOfSpeech.VERB: + case PartOfSpeech.X: + default: + break; + } + + if (!forReference.ContainsKey(token.POS)) + forReference.Add(token.POS, new List() { token.Value.ToLower() }); + else + { + if (!forReference[token.POS].Contains(token.Value.ToLower())) + forReference[token.POS].Add(token.Value.ToLower()); + } + } + } + } + + private void BasicParser(List
allData) + { + foreach (var item in allData) + { + string[] seperators = { ", ", ". ", "! ", "? ", ": ", "; ", " " }; + + string v1 = item.Title.Replace('(', ' ') + .Replace(')', ' ') + .Replace('{', ' ') + .Replace('}', ' ') + .Replace('[', ' ') + .Replace(']', ' '); + foreach (var word in v1.Split(seperators, StringSplitOptions.RemoveEmptyEntries)) + { + if (!string.IsNullOrEmpty(word) && !string.IsNullOrWhiteSpace(word) && !word.ToCharArray().All(s => !char.IsLetterOrDigit(s))) + { + if (commonlyOccuringWords.ContainsKey(word)) + commonlyOccuringWords[word]++; + else + commonlyOccuringWords.Add(word, 1); + } + } + + string v2 = item.RawDescription.Replace('(', ' ') + .Replace(')', ' ') + .Replace('{', ' ') + .Replace('}', ' ') + .Replace('[', ' ') + .Replace(']', ' '); + + foreach (var word in v2.Split(seperators, StringSplitOptions.RemoveEmptyEntries)) + { + if (!string.IsNullOrEmpty(word) && !string.IsNullOrWhiteSpace(word) && !word.ToCharArray().All(s => !char.IsLetterOrDigit(s))) + { + if (commonlyOccuringWords.ContainsKey(word)) + commonlyOccuringWords[word]++; + else + commonlyOccuringWords.Add(word, 1); + } + } + } + } + + public Dictionary GetCommonlyUsedWords(int count = 0) + { + bool resultCacheValueFlag = _cache.TryGetValue("commonlyOccuringWords", out object value); + if (!resultCacheValueFlag || value == null) + PopulateRecommendedWords(); + + return _cache.Get>("commonlyOccuringWords").OrderByDescending(s => s.Value).Take(count).ToDictionary(x => x.Key, x => x.Value); + } + + public Dictionary GetCommonlySearchedWords(int count = 0) + { + IQueryable queryable = applicationDbContext.QueryStats.OrderByDescending(s => s.Count).Take(count); + return queryable.ToDictionary(s => s.Query, s => s.Count); + } + + public Dictionary> GetNLPResult() + { + return forReference.ToDictionary(s => s.Key.ToString(), s => s.Value); + } + } +} diff --git a/Library.Encyclopedia.DataAccess/QueryStatsAccess/QueryStatsAdapter.cs b/Library.Encyclopedia.DataAccess/QueryStatsAccess/QueryStatsAdapter.cs new file mode 100644 index 0000000..a659441 --- /dev/null +++ b/Library.Encyclopedia.DataAccess/QueryStatsAccess/QueryStatsAdapter.cs @@ -0,0 +1,31 @@ +using Library.Encyclopedia.Entity.Models; +using Microsoft.EntityFrameworkCore; +using System.Collections.Concurrent; +using System.Threading.Tasks; + +namespace Library.Encyclopedia.DataAccess.QueryStatsAccess +{ + public class QueryStatsAdapter + { + private readonly IApplicationDbContext _dbcontext; + + public QueryStatsAdapter(IApplicationDbContext applicationDbContext) + { + this._dbcontext = applicationDbContext; + } + + public async Task AddQuery(string query) + { + QueryStats queryStats = await _dbcontext.QueryStats.FirstOrDefaultAsync(s => s.Query == query); + if (queryStats == null) + await _dbcontext.QueryStats.AddAsync(new QueryStats { Query = query, Count = 1 }); + else + { + queryStats.Count++; + _dbcontext.QueryStats.Update(queryStats); + } + + await _dbcontext.SaveChanges(); + } + } +} diff --git a/Library.Encyclopedia.Entity/Models/QueryStats.cs b/Library.Encyclopedia.Entity/Models/QueryStats.cs new file mode 100644 index 0000000..52d304f --- /dev/null +++ b/Library.Encyclopedia.Entity/Models/QueryStats.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Library.Encyclopedia.Entity.Models +{ + public class QueryStats + { + public long Id { get; set; } + public string Query { get; set; } + public long Count { get; set; } + } +}