-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
10 changed files
with
316 additions
and
4 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
68 changes: 68 additions & 0 deletions
68
Library.Encyclopedia.API/Controllers/StatisticsController.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| using Library.Encyclopedia.DataAccess; | ||
| using Library.Encyclopedia.DataAccess.QueryStatsAccess; | ||
| using Microsoft.AspNetCore.Mvc; | ||
| using Microsoft.Extensions.Caching.Memory; | ||
| using Microsoft.Extensions.Logging; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace Library.Encyclopedia.API.Controllers | ||
| { | ||
| [ApiController] | ||
| [Route("[controller]")] | ||
| public class StatisticsController : ControllerBase | ||
| { | ||
| private readonly ILogger<StatisticsController> _logger; | ||
| private readonly CommonlyOccuringWordsAdapter commonlyOccuringWordsAdapter; | ||
|
|
||
| public StatisticsController(ILogger<StatisticsController> logger, IApplicationDbContext applicationDbContext, IMemoryCache memoryCache) | ||
| { | ||
| _logger = logger; | ||
| this.commonlyOccuringWordsAdapter = new CommonlyOccuringWordsAdapter(applicationDbContext, memoryCache); | ||
| } | ||
|
|
||
| [HttpGet("GetRecommendedWords")] | ||
| public IActionResult GetCommonlyOccuringWordsStats(int count = 10) | ||
| { | ||
| try | ||
| { | ||
| return Ok(commonlyOccuringWordsAdapter.GetCommonlyUsedWords(count)); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| _logger.LogError(ex, $"an error has occured {ex.Message}"); | ||
| throw; | ||
| } | ||
| } | ||
|
|
||
| [HttpGet("GetPopularWords")] | ||
| public IActionResult GetMostSearchedWordStats(int count = 10) | ||
| { | ||
| try | ||
| { | ||
| return Ok(commonlyOccuringWordsAdapter.GetCommonlySearchedWords(count)); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| _logger.LogError(ex, $"an error has occured {ex.Message}"); | ||
| throw; | ||
| } | ||
| } | ||
|
|
||
| [HttpGet("GetNLPResult")] | ||
| public IActionResult GetNLPResult() | ||
| { | ||
| try | ||
| { | ||
| return Ok(commonlyOccuringWordsAdapter.GetNLPResult()); | ||
| } | ||
| catch (Exception ex) | ||
| { | ||
| _logger.LogError(ex, $"an error has occured {ex.Message}"); | ||
| throw; | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
175 changes: 175 additions & 0 deletions
175
Library.Encyclopedia.DataAccess/QueryStatsAccess/CommonlyOccuringWordsAdapter.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,175 @@ | ||
| using Catalyst; | ||
| using Catalyst.Models; | ||
| using Library.Encyclopedia.Entity.Models; | ||
| using Microsoft.Extensions.Caching.Memory; | ||
| using Mosaik.Core; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Threading; | ||
|
|
||
| namespace Library.Encyclopedia.DataAccess.QueryStatsAccess | ||
| { | ||
| public class CommonlyOccuringWordsAdapter | ||
| { | ||
| private readonly IApplicationDbContext applicationDbContext; | ||
| private Dictionary<string, long> commonlyOccuringWords; | ||
| private Dictionary<PartOfSpeech, List<string>> forReference; | ||
| private IMemoryCache _cache; | ||
|
|
||
| public CommonlyOccuringWordsAdapter(IApplicationDbContext applicationDbContext, IMemoryCache cache) | ||
| { | ||
| commonlyOccuringWords = new Dictionary<string, long>(); | ||
| forReference = new Dictionary<PartOfSpeech, List<string>>(); | ||
| this.applicationDbContext = applicationDbContext; | ||
| this._cache = cache; | ||
| } | ||
|
|
||
| private void PopulateRecommendedWords() | ||
| { | ||
| commonlyOccuringWords = new Dictionary<string, long>(); | ||
| forReference = new Dictionary<PartOfSpeech, List<string>>(); | ||
|
|
||
| var allData = applicationDbContext.Main.ToList(); | ||
|
|
||
| //BasicParser(allData); | ||
| AdvancedParser(allData); | ||
|
|
||
| var cacheEntryOptions = new MemoryCacheEntryOptions() | ||
| .SetSlidingExpiration(TimeSpan.FromHours(1)); | ||
|
|
||
| if (commonlyOccuringWords != null) | ||
| { | ||
| _cache.Set("commonlyOccuringWords", commonlyOccuringWords, cacheEntryOptions); | ||
| } | ||
| } | ||
|
|
||
| private void AdvancedParser(List<Main> allData) | ||
| { | ||
| English.Register(); | ||
| Storage.Current = new DiskStorage("catalyst-models"); | ||
| var nlp = Pipeline.For(Language.English); | ||
|
|
||
| foreach (var item in allData) | ||
| { | ||
| ProcessText(nlp, item.Title, item.Id); | ||
| ProcessText(nlp, item.RawDescription, item.Id); | ||
| } | ||
| } | ||
|
|
||
| private void ProcessText(Pipeline nlp, string text, Guid id) | ||
| { | ||
| var doc = new Document(text, Language.English); | ||
| nlp.ProcessSingle(doc); | ||
|
|
||
| foreach (var sentence in doc) | ||
| { | ||
| foreach (var token in sentence) | ||
| { | ||
| switch (token.POS) | ||
| { | ||
| case PartOfSpeech.PROPN: | ||
| case PartOfSpeech.NOUN: | ||
| case PartOfSpeech.ADJ: | ||
| case PartOfSpeech.ADV: | ||
| case PartOfSpeech.NUM: | ||
| if (token.Value.Length >= 3) | ||
| { | ||
| if (commonlyOccuringWords.ContainsKey(token.Value)) | ||
| commonlyOccuringWords[token.Value]++; | ||
| else | ||
| commonlyOccuringWords.Add(token.Value, 1); | ||
| } | ||
| break; | ||
| case PartOfSpeech.NONE: | ||
| case PartOfSpeech.ADP: | ||
| case PartOfSpeech.AUX: | ||
| case PartOfSpeech.CCONJ: | ||
| case PartOfSpeech.DET: | ||
| case PartOfSpeech.INTJ: | ||
| case PartOfSpeech.PART: | ||
| case PartOfSpeech.PRON: | ||
| case PartOfSpeech.PUNCT: | ||
| case PartOfSpeech.SCONJ: | ||
| case PartOfSpeech.SYM: | ||
| case PartOfSpeech.VERB: | ||
| case PartOfSpeech.X: | ||
| default: | ||
| break; | ||
| } | ||
|
|
||
| if (!forReference.ContainsKey(token.POS)) | ||
| forReference.Add(token.POS, new List<string>() { token.Value.ToLower() }); | ||
| else | ||
| { | ||
| if (!forReference[token.POS].Contains(token.Value.ToLower())) | ||
| forReference[token.POS].Add(token.Value.ToLower()); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private void BasicParser(List<Main> allData) | ||
| { | ||
| foreach (var item in allData) | ||
| { | ||
| string[] seperators = { ", ", ". ", "! ", "? ", ": ", "; ", " " }; | ||
|
|
||
| string v1 = item.Title.Replace('(', ' ') | ||
| .Replace(')', ' ') | ||
| .Replace('{', ' ') | ||
| .Replace('}', ' ') | ||
| .Replace('[', ' ') | ||
| .Replace(']', ' '); | ||
| foreach (var word in v1.Split(seperators, StringSplitOptions.RemoveEmptyEntries)) | ||
| { | ||
| if (!string.IsNullOrEmpty(word) && !string.IsNullOrWhiteSpace(word) && !word.ToCharArray().All(s => !char.IsLetterOrDigit(s))) | ||
| { | ||
| if (commonlyOccuringWords.ContainsKey(word)) | ||
| commonlyOccuringWords[word]++; | ||
| else | ||
| commonlyOccuringWords.Add(word, 1); | ||
| } | ||
| } | ||
|
|
||
| string v2 = item.RawDescription.Replace('(', ' ') | ||
| .Replace(')', ' ') | ||
| .Replace('{', ' ') | ||
| .Replace('}', ' ') | ||
| .Replace('[', ' ') | ||
| .Replace(']', ' '); | ||
|
|
||
| foreach (var word in v2.Split(seperators, StringSplitOptions.RemoveEmptyEntries)) | ||
| { | ||
| if (!string.IsNullOrEmpty(word) && !string.IsNullOrWhiteSpace(word) && !word.ToCharArray().All(s => !char.IsLetterOrDigit(s))) | ||
| { | ||
| if (commonlyOccuringWords.ContainsKey(word)) | ||
| commonlyOccuringWords[word]++; | ||
| else | ||
| commonlyOccuringWords.Add(word, 1); | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public Dictionary<string, long> GetCommonlyUsedWords(int count = 0) | ||
| { | ||
| bool resultCacheValueFlag = _cache.TryGetValue("commonlyOccuringWords", out object value); | ||
| if (!resultCacheValueFlag || value == null) | ||
| PopulateRecommendedWords(); | ||
|
|
||
| return _cache.Get<Dictionary<string, long>>("commonlyOccuringWords").OrderByDescending(s => s.Value).Take(count).ToDictionary(x => x.Key, x => x.Value); | ||
| } | ||
|
|
||
| public Dictionary<string, long> GetCommonlySearchedWords(int count = 0) | ||
| { | ||
| IQueryable<QueryStats> queryable = applicationDbContext.QueryStats.OrderByDescending(s => s.Count).Take(count); | ||
| return queryable.ToDictionary(s => s.Query, s => s.Count); | ||
| } | ||
|
|
||
| public Dictionary<string, List<string>> GetNLPResult() | ||
| { | ||
| return forReference.ToDictionary(s => s.Key.ToString(), s => s.Value); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.