From 283ab8e7184f0be1e629ed991575bffff23cc6ef Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sun, 7 Jul 2024 23:07:41 +0200 Subject: [PATCH 01/12] Update IndexJsonBase.cs, OutputWriteUtil.cs, and Converter.cs --- .../Objects/Json/Indexes/IndexJsonBase.cs | 3 +- .../Utils/Output/OutputWriteUtil.cs | 2 +- .../Utils/Transformer/ParserNS/Converter.cs | 49 ------------------- 3 files changed, 2 insertions(+), 52 deletions(-) delete mode 100644 PoliNetwork.Graduatorie.Parser/Utils/Transformer/ParserNS/Converter.cs diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs index a21bbaea..1a9eb5c2 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs @@ -3,7 +3,6 @@ using Newtonsoft.Json; using Newtonsoft.Json.Serialization; using PoliNetwork.Graduatorie.Common.Data; -using PoliNetwork.Graduatorie.Common.Objects; using PoliNetwork.Graduatorie.Parser.Objects.Json.Indexes.Specific; using PoliNetwork.Graduatorie.Parser.Objects.RankingNS; @@ -24,7 +23,7 @@ public void Write(string outFolder, string pathFile) File.WriteAllText(mainJsonPath, mainJsonString); } - public static void WriteAllIndexes(RankingsSet rankingsSet, string outFolder, ArgsConfig argsConfig) + public static void WriteAllIndexes(RankingsSet rankingsSet, string outFolder) { //now let's write each single different index BySchoolYearJson.From(rankingsSet).Write(outFolder, BySchoolYearJson.CustomPath); diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Output/OutputWriteUtil.cs b/PoliNetwork.Graduatorie.Parser/Utils/Output/OutputWriteUtil.cs index 1f422b5e..a6c8f8e4 100644 --- a/PoliNetwork.Graduatorie.Parser/Utils/Output/OutputWriteUtil.cs +++ b/PoliNetwork.Graduatorie.Parser/Utils/Output/OutputWriteUtil.cs @@ -25,7 +25,7 @@ public void SaveOutputs(RankingsSet rankingsSet, DateFound dateFound) var outFolder = Path.Join(_config.DataFolder, Constants.OutputFolder); rankingsSet.WriteAllRankings(outFolder, _config.ForceReparsing); - IndexJsonBase.WriteAllIndexes(rankingsSet, outFolder, _config); + IndexJsonBase.WriteAllIndexes(rankingsSet, outFolder); StatsJson.From(rankingsSet).Write(outFolder, _config); HashMatricoleWrite.From(rankingsSet).Write(outFolder); diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Transformer/ParserNS/Converter.cs b/PoliNetwork.Graduatorie.Parser/Utils/Transformer/ParserNS/Converter.cs deleted file mode 100644 index fc32e7a4..00000000 --- a/PoliNetwork.Graduatorie.Parser/Utils/Transformer/ParserNS/Converter.cs +++ /dev/null @@ -1,49 +0,0 @@ -#region - -using PoliNetwork.Graduatorie.Common.Objects; -using PoliNetwork.Graduatorie.Parser.Objects; -using PoliNetwork.Graduatorie.Parser.Objects.Tables.Course; -using PoliNetwork.Graduatorie.Parser.Objects.Tables.Merit; - -#endregion - -namespace PoliNetwork.Graduatorie.Parser.Utils.Transformer.ParserNS; - -public static class Converter -{ - public static StudentResult FromMeritTableToStudentResult(MeritTableRow row) - { - var rowCanEnroll = row.CanEnroll ?? false; - var rowCanEnrollInto = rowCanEnroll ? row.CanEnrollInto : null; - return new StudentResult - { - Id = row.Id, - Ofa = row.Ofa, - Result = row.Result, - BirthDate = null, - PositionAbsolute = row.Position, - PositionCourse = null, - SectionsResults = null, - EnglishCorrectAnswers = null, - EnrollType = EnrollType.From(rowCanEnrollInto, rowCanEnroll) - }; - } - - - public static StudentResult FromCourseTableRowToStudentResult(CourseTableRow row, Table course) - { - var rowCanEnroll = row.CanEnroll ?? false; - return new StudentResult - { - Id = row.Id, - Ofa = row.Ofa, - Result = row.Result, - BirthDate = row.BirthDate, - EnrollType = EnrollType.From(course.CourseTitle, rowCanEnroll), - PositionAbsolute = null, - PositionCourse = row.Position, - SectionsResults = row.SectionsResults, - EnglishCorrectAnswers = row.EnglishCorrectAnswers - }; - } -} \ No newline at end of file From 9f33661da7457b7f2d53ea0f274a27a9d127890b Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sun, 7 Jul 2024 23:07:59 +0200 Subject: [PATCH 02/12] Update Lib_CSharp --- Lib_CSharp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib_CSharp b/Lib_CSharp index 6b7ed8c0..e53d6712 160000 --- a/Lib_CSharp +++ b/Lib_CSharp @@ -1 +1 @@ -Subproject commit 6b7ed8c04f1728570c2f7575bc1ceab2b713f04b +Subproject commit e53d671278c21d349835af31394d7bc42be49e7b From 7c0d2df0afb670e8188f259881a59745a3a36eff Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sun, 7 Jul 2024 23:08:31 +0200 Subject: [PATCH 03/12] Update Ranking.cs --- PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs index 56fbc52b..a438cf74 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs @@ -183,4 +183,9 @@ public int GetHashWithoutLastUpdate() return i; } + + public override bool Equals(object obj) + { + return Equals(obj as Ranking); + } } \ No newline at end of file From fa8fe10e48eb354cc2feb859af587493a28f0e06 Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sun, 7 Jul 2024 23:09:00 +0200 Subject: [PATCH 04/12] Update BySchoolYearJson.cs --- .../Objects/Json/Indexes/Specific/BySchoolYearJson.cs | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearJson.cs index 2798b366..f37698d4 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearJson.cs @@ -90,15 +90,6 @@ public static RankingsSet GetAndParse(string dataFolder) public List GetRankings(string outFolder) { - List rankings = new(); - - foreach (var singleCourseJson in All) - { - var fullPath = singleCourseJson.GetFullPath(outFolder); - var ranking = Ranking.FromJson(fullPath); - if (ranking != null) rankings.Add(ranking); - } - - return rankings; + return All.Select(singleCourseJson => singleCourseJson.GetFullPath(outFolder)).Select(Ranking.FromJson).OfType().ToList(); } } \ No newline at end of file From 4a0b2871b069747ebc6089c47166a0afca9d1a05 Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sun, 7 Jul 2024 23:09:20 +0200 Subject: [PATCH 05/12] Update StatsSchool.cs and StatsYear.cs --- .../Objects/Json/Stats/StatsSchool.cs | 2 +- .../Objects/Json/Stats/StatsYear.cs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSchool.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSchool.cs index 428a772f..f29f163b 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSchool.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSchool.cs @@ -20,7 +20,7 @@ public class StatsSchool public static StatsSchool From(List pRankings) { var statsSchool = new StatsSchool(); - var rankings = pRankings.Where(r => r.Year != null && r.School != null).ToList(); + var rankings = pRankings.Where(r => r is { Year: not null, School: not null }).ToList(); statsSchool.NumStudents = rankings.Select(x => (x.RankingSummary ?? x.CreateSummary()).HowManyStudents ?? 0).Sum(); diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsYear.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsYear.cs index c3a80552..38c470dc 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsYear.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsYear.cs @@ -25,11 +25,11 @@ public class StatsYear public static StatsYear From(List rankings) { - var statsYear = new StatsYear(); - - statsYear.Year = rankings.First(r => r.Year != null).Year!.Value; // just hilarious - statsYear.NumStudents = - rankings.Select(r => (r.RankingSummary ?? r.CreateSummary()).HowManyStudents ?? 0).Sum(); // this ?? is crazy + var statsYear = new StatsYear + { + Year = rankings.First(r => r.Year != null).Year!.Value, // just hilarious + NumStudents = rankings.Select(r => (r.RankingSummary ?? r.CreateSummary()).HowManyStudents ?? 0).Sum() // this ?? is crazy + }; var bySchool = rankings.Where(r => r.School != null).GroupBy(r => r.School!.Value); foreach (var schoolGroup in bySchool) From 9f1d7711bff5856b9fa5e45d952c1fbacbee9eec Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sun, 7 Jul 2024 23:09:44 +0200 Subject: [PATCH 06/12] Update Ranking.cs --- .../Objects/RankingNS/Ranking.cs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs index a438cf74..2748b487 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs @@ -177,15 +177,19 @@ public int GetHashWithoutLastUpdate() i = ByCourse.Aggregate(i, (current, variable) => { var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); - var iList = hashWithoutLastUpdate; - return current ^ iList; + return current ^ hashWithoutLastUpdate; }); return i; } - public override bool Equals(object obj) + public override bool Equals(object? obj) { return Equals(obj as Ranking); } + + public override int GetHashCode() + { + return this.GetHashWithoutLastUpdate(); + } } \ No newline at end of file From f853dd90744f08d4b3ebd42afc4251dc8da2e169 Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sun, 7 Jul 2024 23:09:57 +0200 Subject: [PATCH 07/12] Update RankingsSet.cs --- .../Objects/RankingNS/RankingsSet.cs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingsSet.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingsSet.cs index 908d396a..91bb6cc7 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingsSet.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingsSet.cs @@ -11,14 +11,8 @@ namespace PoliNetwork.Graduatorie.Parser.Objects.RankingNS; [JsonObject(MemberSerialization.Fields, NamingStrategyType = typeof(CamelCaseNamingStrategy))] public class RankingsSet { - public DateTime? LastUpdate; - public List Rankings; - - public RankingsSet() - { - Rankings = new List(); - LastUpdate = DateTime.UtcNow; - } + public DateTime? LastUpdate = DateTime.UtcNow; + public List Rankings = new(); public void Merge(RankingsSet newSet) { From 01a5871798ae03e045c251afa42d72f67c80769a Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sun, 7 Jul 2024 23:10:32 +0200 Subject: [PATCH 08/12] Update RankingsSet.cs, CourseTable.cs, and HashMatricoleWrite.cs --- .../Objects/RankingNS/RankingsSet.cs | 10 ++++------ .../Objects/Tables/Course/CourseTable.cs | 3 +-- .../Utils/Output/HashMatricoleWrite.cs | 2 +- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingsSet.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingsSet.cs index 91bb6cc7..1eb47859 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingsSet.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingsSet.cs @@ -19,13 +19,11 @@ public void Merge(RankingsSet newSet) foreach (var ranking in newSet.Rankings) { var alreadyPresent = Rankings.Any(v => v.IsSimilarTo(ranking)); - if (!alreadyPresent) - { - Rankings.Add(ranking); + if (alreadyPresent) continue; + Rankings.Add(ranking); - if (LastUpdate == null || ranking.LastUpdate.Date > LastUpdate?.Date) - LastUpdate = ranking.LastUpdate; - } + if (LastUpdate == null || ranking.LastUpdate.Date > LastUpdate?.Date) + LastUpdate = ranking.LastUpdate; } } diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs b/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs index 17aac09b..bb599a7e 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs @@ -41,7 +41,6 @@ public string GetFixedLocation() // esempio: Urbanistica 2022 ha un solo corso senza location, ma anche quello // deve comparire nella lista // fix: se un corso non ha location, si inserisce un valore 0 - if (string.IsNullOrEmpty(Location)) return Constants.LocationPlaceholder; - return Location; + return string.IsNullOrEmpty(Location) ? Constants.LocationPlaceholder : Location; } } \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs b/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs index 86d4ae06..7657d974 100644 --- a/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs +++ b/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs @@ -43,7 +43,7 @@ public void Write(string outFolder) private static IdsDict GetIdsDict(RankingsSet rankingsSet) { - var dictionary = new SortedDictionary(); + var dictionary = new IdsDict(); foreach (var ranking in rankingsSet.Rankings) { var byMeritRows = ranking.ByMerit?.Rows; From 840569e73f4ecf59fdc470df1c82ba06c053816f Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sun, 7 Jul 2024 23:11:48 +0200 Subject: [PATCH 09/12] Update BySchoolYearCourseJson.cs, StatsSchool.cs, Ranking.cs, and 2 more files --- .../Json/Indexes/Specific/BySchoolYearCourseJson.cs | 2 +- .../Objects/Json/Stats/StatsSchool.cs | 2 +- .../Objects/RankingNS/Ranking.cs | 11 +++++------ .../Utils/Output/HashMatricoleWrite.cs | 10 +++++----- .../Utils/Web/ScraperOutput.cs | 6 +++--- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs index 51613387..ff5f0e10 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs @@ -67,7 +67,7 @@ private static YearsDict GetYearsDict(IEnumerable> byYea return yearsDict; } - private static CoursesDict GetCoursesDict(IGrouping yearGroup) + private static CoursesDict GetCoursesDict(IEnumerable yearGroup) { var coursesDict = new CoursesDict(); diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSchool.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSchool.cs index f29f163b..0a80f736 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSchool.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsSchool.cs @@ -17,7 +17,7 @@ public class StatsSchool public List List = new(); public int NumStudents; - public static StatsSchool From(List pRankings) + public static StatsSchool From(IEnumerable pRankings) { var statsSchool = new StatsSchool(); var rankings = pRankings.Where(r => r is { Year: not null, School: not null }).ToList(); diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs index 2748b487..d2901b16 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs @@ -148,11 +148,10 @@ public void WriteAsJson(string outFolder, bool forceReparse = false) var savedRanking = FromJson(fullPath); var equalsSaved = savedRanking != null && Equals(savedRanking); - if (forceReparse || equalsSaved || savedRanking == null) - { - var rankingJsonString = JsonConvert.SerializeObject(this, Culture.JsonSerializerSettings); - File.WriteAllText(fullPath, rankingJsonString); - } + if (!forceReparse && !equalsSaved && savedRanking != null) return; + + var rankingJsonString = JsonConvert.SerializeObject(this, Culture.JsonSerializerSettings); + File.WriteAllText(fullPath, rankingJsonString); } /*** @@ -190,6 +189,6 @@ public override bool Equals(object? obj) public override int GetHashCode() { - return this.GetHashWithoutLastUpdate(); + return GetHashWithoutLastUpdate(); } } \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs b/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs index 7657d974..4cd6a489 100644 --- a/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs +++ b/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs @@ -12,21 +12,21 @@ namespace PoliNetwork.Graduatorie.Parser.Utils.Output; public class HashMatricoleWrite { - internal const string FolderName = "hashMatricole"; - public IdsDict IdsDict = new(); + private const string FolderName = "hashMatricole"; + private IdsDict _idsDict = new(); public static HashMatricoleWrite From(RankingsSet rankingsSet) { return new HashMatricoleWrite { - IdsDict = GetIdsDict(rankingsSet) + _idsDict = GetIdsDict(rankingsSet) }; } public void Write(string outFolder) { - Console.WriteLine($"[INFO] Students with id are {IdsDict.Keys.Count}"); + Console.WriteLine($"[INFO] Students with id are {_idsDict.Keys.Count}"); var groupsDict = GetGroupsDict(); var hashMatricoleFolder = Path.Join(outFolder, FolderName); @@ -81,7 +81,7 @@ private static IdsDict GetIdsDict(RankingsSet rankingsSet) private SortedDictionary GetGroupsDict() { var groupsDict = new SortedDictionary(); - var groups = IdsDict.GroupBy(pair => pair.Key[..2]); + var groups = _idsDict.GroupBy(pair => pair.Key[..2]); foreach (var group in groups) { diff --git a/PoliNetwork.Graduatorie.Scraper/Utils/Web/ScraperOutput.cs b/PoliNetwork.Graduatorie.Scraper/Utils/Web/ScraperOutput.cs index 0313a170..d82e7ea4 100644 --- a/PoliNetwork.Graduatorie.Scraper/Utils/Web/ScraperOutput.cs +++ b/PoliNetwork.Graduatorie.Scraper/Utils/Web/ScraperOutput.cs @@ -32,11 +32,11 @@ private static List GetSaved(string dataFolder) { // consider to handle them Console.WriteLine($"[ERROR] Can't validate the ScraperOutput file ({filePath})"); - return new(); + return new List(); } } - private static List GetLines(string filePath) + private static IEnumerable GetLines(string filePath) { try { @@ -46,7 +46,7 @@ private static List GetLines(string filePath) { Console.WriteLine(exception); Console.WriteLine($"[ERROR] Can't read the ScraperOutput file ({filePath})"); - return new(); + return new List(); } } From 3c834571b25a024cd4bea16b842b9cf6e31bd8be Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sun, 7 Jul 2024 23:12:34 +0200 Subject: [PATCH 10/12] Update EnrollType.cs, IndexJsonBase.cs, BySchoolYearJson.cs, and 4 more files --- .../Objects/EnrollType.cs | 2 +- .../Objects/Json/Indexes/IndexJsonBase.cs | 2 +- .../Json/Indexes/Specific/BySchoolYearJson.cs | 7 ++++--- .../Json/Indexes/Specific/ByYearSchoolJson.cs | 10 +++++----- .../Objects/Json/Stats/StatsJson.cs | 6 ++---- .../Utils/Output/OutputWriteUtil.cs | 2 +- .../Utils/Output/RankingSummaryStudent.cs | 15 ++++++++------- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/PoliNetwork.Graduatorie.Common/Objects/EnrollType.cs b/PoliNetwork.Graduatorie.Common/Objects/EnrollType.cs index 2fb946ec..09d44d99 100644 --- a/PoliNetwork.Graduatorie.Common/Objects/EnrollType.cs +++ b/PoliNetwork.Graduatorie.Common/Objects/EnrollType.cs @@ -23,7 +23,7 @@ public int GetHashWithoutLastUpdate() return i; } - + public static EnrollType From(string? rowCanEnrollInto, bool rowCanEnroll) { if (rowCanEnroll == false) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs index 1a9eb5c2..f685e341 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs @@ -22,7 +22,7 @@ public void Write(string outFolder, string pathFile) var mainJsonString = JsonConvert.SerializeObject(this, Culture.JsonSerializerSettings); File.WriteAllText(mainJsonPath, mainJsonString); } - + public static void WriteAllIndexes(RankingsSet rankingsSet, string outFolder) { //now let's write each single different index diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearJson.cs index f37698d4..fd5a1c5f 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearJson.cs @@ -18,9 +18,9 @@ namespace PoliNetwork.Graduatorie.Parser.Objects.Json.Indexes.Specific; public class BySchoolYearJson : IndexJsonBase { internal const string CustomPath = "bySchoolYear.json"; + public List All = new(); // decide whether to include it in the json serialization public SchoolsDict Schools = new(); - public List All = new(); // decide whether to include it in the json serialization public static BySchoolYearJson From(RankingsSet set) { @@ -30,7 +30,7 @@ public static BySchoolYearJson From(RankingsSet set) .SelectMany(r => r.ToSingleCourseJson()) .DistinctBy(r => new { r.Id }) .ToList(); - + list.Sort(); mainJson.All = list; @@ -90,6 +90,7 @@ public static RankingsSet GetAndParse(string dataFolder) public List GetRankings(string outFolder) { - return All.Select(singleCourseJson => singleCourseJson.GetFullPath(outFolder)).Select(Ranking.FromJson).OfType().ToList(); + return All.Select(singleCourseJson => singleCourseJson.GetFullPath(outFolder)).Select(Ranking.FromJson) + .OfType().ToList(); } } \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/ByYearSchoolJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/ByYearSchoolJson.cs index a5b153ef..d1e973b9 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/ByYearSchoolJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/ByYearSchoolJson.cs @@ -17,22 +17,22 @@ namespace PoliNetwork.Graduatorie.Parser.Objects.Json.Indexes.Specific; public class ByYearSchoolJson : IndexJsonBase { internal const string CustomPath = "byYearSchool.json"; + public List All = new(); // decide whether to include it in the json serialization public YearsDict Years = new(); - public List All = new(); // decide whether to include it in the json serialization public static ByYearSchoolJson From(RankingsSet set) { var mainJson = new ByYearSchoolJson { LastUpdate = set.LastUpdate }; - + var list = set.Rankings .SelectMany(r => r.ToSingleCourseJson()).ToList() .DistinctBy(r => new { r.Id }) .ToList(); - + list.Sort(); mainJson.All = list; - + // group rankings by year var byYear = set.Rankings.Where(r => r.Year != null).GroupBy(r => r.Year!.Value); foreach (var yearGroup in byYear) @@ -57,7 +57,7 @@ private static SchoolsDict GetSchoolsDict(IEnumerable x.Link) .OrderBy(r => r.Id) .ToList(); - + schoolsDict.Add(schoolGroup.Key, filenames); } diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsJson.cs index ca80c939..1befb2b1 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsJson.cs @@ -5,6 +5,7 @@ using Newtonsoft.Json.Serialization; using PoliNetwork.Graduatorie.Common.Objects; using PoliNetwork.Graduatorie.Parser.Objects.RankingNS; + // ReSharper disable CanSimplifyDictionaryLookupWithTryAdd #endregion @@ -41,10 +42,7 @@ public void Write(string outFolder, ArgsConfig argsConfig) var statsFolderPath = Path.Join(outFolder, StatsFolderName); if (!Directory.Exists(statsFolderPath)) Directory.CreateDirectory(statsFolderPath); - foreach (var yearStats in Stats.Values) - { - yearStats.Write(statsFolderPath, argsConfig); - } + foreach (var yearStats in Stats.Values) yearStats.Write(statsFolderPath, argsConfig); } public int GetHashWithoutLastUpdate() diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Output/OutputWriteUtil.cs b/PoliNetwork.Graduatorie.Parser/Utils/Output/OutputWriteUtil.cs index a6c8f8e4..3e294db8 100644 --- a/PoliNetwork.Graduatorie.Parser/Utils/Output/OutputWriteUtil.cs +++ b/PoliNetwork.Graduatorie.Parser/Utils/Output/OutputWriteUtil.cs @@ -23,7 +23,7 @@ public OutputWriteUtil(ArgsConfig argsConfig) public void SaveOutputs(RankingsSet rankingsSet, DateFound dateFound) { var outFolder = Path.Join(_config.DataFolder, Constants.OutputFolder); - + rankingsSet.WriteAllRankings(outFolder, _config.ForceReparsing); IndexJsonBase.WriteAllIndexes(rankingsSet, outFolder); StatsJson.From(rankingsSet).Write(outFolder, _config); diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Output/RankingSummaryStudent.cs b/PoliNetwork.Graduatorie.Parser/Utils/Output/RankingSummaryStudent.cs index 2a0299f8..52e783ef 100644 --- a/PoliNetwork.Graduatorie.Parser/Utils/Output/RankingSummaryStudent.cs +++ b/PoliNetwork.Graduatorie.Parser/Utils/Output/RankingSummaryStudent.cs @@ -40,6 +40,14 @@ public RankingSummaryStudent(string? phase, SchoolEnum? school, int? year, Ranki Url = url; } + public bool Equals(RankingSummaryStudent? other) + { + if (ReferenceEquals(null, other)) return false; + if (ReferenceEquals(this, other)) return true; + return Course == other.Course && Phase == other.Phase && School == other.School && Equals(Url, other.Url) && + Year == other.Year; + } + public int Compare(RankingSummaryStudent o) { var i = (Year ?? 0) - (o.Year ?? 0); @@ -70,13 +78,6 @@ public override bool Equals(object? obj) Year == rankingSummaryStudent.Year && equals && Course == rankingSummaryStudent.Course; } - public bool Equals(RankingSummaryStudent? other) - { - if (ReferenceEquals(null, other)) return false; - if (ReferenceEquals(this, other)) return true; - return Course == other.Course && Phase == other.Phase && School == other.School && Equals(Url, other.Url) && Year == other.Year; - } - public override int GetHashCode() { return HashCode.Combine(Course, Phase, School, Url, Year); From 374153ff6012ab046fd9e521fae16768ae35cdb7 Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sun, 7 Jul 2024 23:12:44 +0200 Subject: [PATCH 11/12] Update BySchoolYearCourseJson.cs, SingleCourseJson.cs, StatsYear.cs, and 5 more files --- .../Specific/BySchoolYearCourseJson.cs | 8 ++-- .../Objects/Json/SingleCourseJson.cs | 34 ++++++++-------- .../Objects/Json/Stats/StatsYear.cs | 13 +++--- .../Objects/RankingNS/Ranking.cs | 40 +++++++++---------- .../Objects/RankingNS/RankingsSet.cs | 4 +- .../Objects/Tables/Course/CourseTable.cs | 4 +- .../Utils/Output/HashMatricoleWrite.cs | 16 +++----- .../Utils/Web/ScraperOutput.cs | 3 +- 8 files changed, 60 insertions(+), 62 deletions(-) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs index ff5f0e10..d41629b2 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs @@ -1,9 +1,11 @@ #region + using System.Diagnostics; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; using PoliNetwork.Graduatorie.Common.Enums; using PoliNetwork.Graduatorie.Parser.Objects.RankingNS; + #endregion namespace PoliNetwork.Graduatorie.Parser.Objects.Json.Indexes.Specific; @@ -20,20 +22,20 @@ namespace PoliNetwork.Graduatorie.Parser.Objects.Json.Indexes.Specific; public class BySchoolYearCourseJson : IndexJsonBase { internal const string CustomPath = "bySchoolYearCourse.json"; + public List All = new(); // decide whether to include it in the json serialization //keys: school, year, course, location public SchoolsDict Schools = new(); - public List All = new(); // decide whether to include it in the json serialization public static BySchoolYearCourseJson From(RankingsSet set) { var mainJson = new BySchoolYearCourseJson { LastUpdate = set.LastUpdate }; - + var list = set.Rankings .SelectMany(r => r.ToSingleCourseJson()) .DistinctBy(r => new { r.Id, r.Location }) .ToList(); - + list.Sort(); mainJson.All = list; diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs index 99938652..ad6842e4 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs @@ -22,25 +22,10 @@ public class SingleCourseJson : IComparable public SchoolEnum? School; public int? Year; - public static SingleCourseJson From(Ranking ranking, CourseTable? course) - { - var basePath = $"{ranking.School}/{ranking.Year}/"; // "Ingegneria/2023" - return new SingleCourseJson - { - Link = ranking.GetFilename(), - Id = ranking.GetId(), - BasePath = basePath, - Year = ranking.Year, - School = ranking.School, - Location = course?.Location, - RankingOrder = ranking.RankingOrder - }; - } - public int CompareTo(SingleCourseJson? singleCourseJson) { if (singleCourseJson == null) return 1; - + if (Year != singleCourseJson.Year) return (Year ?? -1) < (singleCourseJson.Year ?? -1) ? -1 : 1; @@ -60,7 +45,22 @@ public int CompareTo(SingleCourseJson? singleCourseJson) return 0; } - + + public static SingleCourseJson From(Ranking ranking, CourseTable? course) + { + var basePath = $"{ranking.School}/{ranking.Year}/"; // "Ingegneria/2023" + return new SingleCourseJson + { + Link = ranking.GetFilename(), + Id = ranking.GetId(), + BasePath = basePath, + Year = ranking.Year, + School = ranking.School, + Location = course?.Location, + RankingOrder = ranking.RankingOrder + }; + } + public int GetHashWithoutLastUpdate() { var hashWithoutLastUpdate = Link?.GetHashCode() ?? "Link".GetHashCode(); diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsYear.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsYear.cs index 38c470dc..53d4805a 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsYear.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Stats/StatsYear.cs @@ -7,6 +7,7 @@ using PoliNetwork.Graduatorie.Common.Enums; using PoliNetwork.Graduatorie.Common.Objects; using PoliNetwork.Graduatorie.Parser.Objects.RankingNS; + // ReSharper disable CanSimplifyDictionaryLookupWithTryAdd #endregion @@ -19,8 +20,8 @@ namespace PoliNetwork.Graduatorie.Parser.Objects.Json.Stats; [JsonObject(MemberSerialization.Fields, NamingStrategyType = typeof(CamelCaseNamingStrategy))] public class StatsYear { - public SchoolsDict Schools = new(); public int NumStudents; + public SchoolsDict Schools = new(); public int Year; public static StatsYear From(List rankings) @@ -28,7 +29,8 @@ public static StatsYear From(List rankings) var statsYear = new StatsYear { Year = rankings.First(r => r.Year != null).Year!.Value, // just hilarious - NumStudents = rankings.Select(r => (r.RankingSummary ?? r.CreateSummary()).HowManyStudents ?? 0).Sum() // this ?? is crazy + NumStudents = rankings.Select(r => (r.RankingSummary ?? r.CreateSummary()).HowManyStudents ?? 0) + .Sum() // this ?? is crazy }; var bySchool = rankings.Where(r => r.School != null).GroupBy(r => r.School!.Value); @@ -36,10 +38,11 @@ public static StatsYear From(List rankings) { var statsSchool = StatsSchool.From(schoolGroup.ToList()); - if (statsYear.Schools.ContainsKey(schoolGroup.Key)) throw new UnreachableException(); // should be impossible, right? + if (statsYear.Schools.ContainsKey(schoolGroup.Key)) + throw new UnreachableException(); // should be impossible, right? statsYear.Schools.Add(schoolGroup.Key, statsSchool); } - + return statsYear; } @@ -62,7 +65,7 @@ private bool ExitIfThereIsntAnUpdate(string fullJsonPath) var savedStats = JsonConvert.DeserializeObject(saved, Culture.JsonSerializerSettings); if (string.IsNullOrEmpty(saved) || savedStats == null) return false; - + var savedHash = savedStats.GetHashWithoutLastUpdate(); var hash = GetHashWithoutLastUpdate(); return savedHash == hash; diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs index d2901b16..ec0e49a4 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs @@ -30,23 +30,6 @@ public class Ranking : IComparable, IEquatable public RankingUrl? Url; public int? Year; - public RankingSummaryStudent GetRankingSummaryStudent() - { - return new RankingSummaryStudent(RankingOrder?.Phase, School, Year, Url); - } - - public static Ranking? FromJson(string fullPath) - { - // if (!File.Exists(fullPath)) return null; - // - // var str = File.ReadAllText(fullPath); - // var ranking = JsonConvert.DeserializeObject(str, Culture.JsonSerializerSettings); - // return ranking; - - // consider merging the two functions at some point - return Utils.Transformer.ParserNS.Parser.ParseJsonRanking(fullPath); - } - public int CompareTo(Ranking? other) { if (ReferenceEquals(this, other)) return 0; @@ -62,6 +45,23 @@ public bool Equals(Ranking? other) return GetHashWithoutLastUpdate() == other.GetHashWithoutLastUpdate(); } + public RankingSummaryStudent GetRankingSummaryStudent() + { + return new RankingSummaryStudent(RankingOrder?.Phase, School, Year, Url); + } + + public static Ranking? FromJson(string fullPath) + { + // if (!File.Exists(fullPath)) return null; + // + // var str = File.ReadAllText(fullPath); + // var ranking = JsonConvert.DeserializeObject(str, Culture.JsonSerializerSettings); + // return ranking; + + // consider merging the two functions at some point + return Utils.Transformer.ParserNS.Parser.ParseJsonRanking(fullPath); + } + public bool IsSimilarTo(Ranking ranking) { @@ -142,18 +142,18 @@ public void WriteAsJson(string outFolder, bool forceReparse = false) { var folderPath = GetBasePath(outFolder); Directory.CreateDirectory(folderPath); - + var fullPath = GetFullPath(outFolder); var savedRanking = FromJson(fullPath); var equalsSaved = savedRanking != null && Equals(savedRanking); if (!forceReparse && !equalsSaved && savedRanking != null) return; - + var rankingJsonString = JsonConvert.SerializeObject(this, Culture.JsonSerializerSettings); File.WriteAllText(fullPath, rankingJsonString); } - + /*** * Ottieni l'hash senza considerare il valore di LastUpdate */ diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingsSet.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingsSet.cs index 1eb47859..d129cfa4 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingsSet.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingsSet.cs @@ -21,8 +21,8 @@ public void Merge(RankingsSet newSet) var alreadyPresent = Rankings.Any(v => v.IsSimilarTo(ranking)); if (alreadyPresent) continue; Rankings.Add(ranking); - - if (LastUpdate == null || ranking.LastUpdate.Date > LastUpdate?.Date) + + if (LastUpdate == null || ranking.LastUpdate.Date > LastUpdate?.Date) LastUpdate = ranking.LastUpdate; } } diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs b/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs index bb599a7e..4e6360fa 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs @@ -31,8 +31,8 @@ public RankingSummaryStudent GetRankingSummaryStudent(Ranking ranking) } /// - /// Get the course location if present, otherwise get the placeholder (constant). - /// Useful for index purposes. + /// Get the course location if present, otherwise get the placeholder (constant). + /// Useful for index purposes. /// /// A string with the location or the placeholder public string GetFixedLocation() diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs b/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs index 4cd6a489..80396565 100644 --- a/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs +++ b/PoliNetwork.Graduatorie.Parser/Utils/Output/HashMatricoleWrite.cs @@ -22,7 +22,7 @@ public static HashMatricoleWrite From(RankingsSet rankingsSet) _idsDict = GetIdsDict(rankingsSet) }; } - + public void Write(string outFolder) { @@ -63,17 +63,14 @@ private static IdsDict GetIdsDict(RankingsSet rankingsSet) foreach (var student in row.Where(studentResult => !string.IsNullOrEmpty(studentResult.Id))) { var id = student.Id!; - + if (!dictionary.ContainsKey(id)) dictionary.Add(id, new StudentHashSummary()); dictionary[id].Merge(student, ranking, courseTable); } } } - foreach (var item in dictionary.Values) - { - item.Sort(); - } + foreach (var item in dictionary.Values) item.Sort(); return dictionary; } @@ -89,11 +86,8 @@ private SortedDictionary GetGroupsDict() var groupVal = group.ToList(); var groupIdsDict = new IdsDict(); - foreach (var (id, studentHashSummary) in groupVal) - { - groupIdsDict.Add(id, studentHashSummary); - } - + foreach (var (id, studentHashSummary) in groupVal) groupIdsDict.Add(id, studentHashSummary); + groupsDict.Add(groupId, groupIdsDict); } diff --git a/PoliNetwork.Graduatorie.Scraper/Utils/Web/ScraperOutput.cs b/PoliNetwork.Graduatorie.Scraper/Utils/Web/ScraperOutput.cs index d82e7ea4..24ca314e 100644 --- a/PoliNetwork.Graduatorie.Scraper/Utils/Web/ScraperOutput.cs +++ b/PoliNetwork.Graduatorie.Scraper/Utils/Web/ScraperOutput.cs @@ -9,7 +9,6 @@ namespace PoliNetwork.Graduatorie.Scraper.Utils.Web; public static class ScraperOutput { - public static List GetWithUrlsFromLocalFileLinks(IEnumerable urls, string dataFolder) { var links = GetSaved(dataFolder); @@ -84,7 +83,7 @@ private static string SelectorUrlWithEndLine(string url) { return url + "\n"; } - + private static string GetFilePath(string dataFolder) { return Path.Join(dataFolder, Constants.OutputLinksFilename); From 120888a5ebe7a7398dda7244f89e794f27bdb2a5 Mon Sep 17 00:00:00 2001 From: Lorenzo Corallo <66379281+lorenzocorallo@users.noreply.github.com> Date: Sun, 7 Jul 2024 23:33:34 +0200 Subject: [PATCH 12/12] fix: check saved ranking before saving (#170) --- PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs index ec0e49a4..2eac3d2f 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs @@ -148,7 +148,7 @@ public void WriteAsJson(string outFolder, bool forceReparse = false) var savedRanking = FromJson(fullPath); var equalsSaved = savedRanking != null && Equals(savedRanking); - if (!forceReparse && !equalsSaved && savedRanking != null) return; + if (!forceReparse && equalsSaved) return; var rankingJsonString = JsonConvert.SerializeObject(this, Culture.JsonSerializerSettings); File.WriteAllText(fullPath, rankingJsonString);