From 3deae4b8121b06a720d750007ba220c5b7a940b6 Mon Sep 17 00:00:00 2001 From: angeousta <132761637+angeousta@users.noreply.github.com> Date: Sat, 6 Jul 2024 10:57:39 +0200 Subject: [PATCH 1/6] Refactor30 (#166) * Update Lib_CSharp * Update SchoolEnum.cs, IndexJsonBase.cs, SingleCourseJson.cs, and 8 more files --------- Co-authored-by: userbot_github --- Lib_CSharp | 2 +- .../Enums/SchoolEnum.cs | 2 +- .../Objects/Json/Indexes/IndexJsonBase.cs | 15 ++++----- .../Objects/Json/SingleCourseJson.cs | 28 ++++++++--------- .../Objects/RankingNS/Ranking.cs | 21 ++++++------- .../Objects/RankingNS/RankingOrder.cs | 31 +++++++++---------- .../Objects/StudentResult.cs | 5 +-- .../Objects/Tables/Merit/MeritTable.cs | 7 ++--- .../Utils/Hashing.cs | 9 +++--- .../Utils/Transformer/ParserNS/Parser.cs | 15 ++++----- .../Utils/Web/Scraper.cs | 7 +++-- .../Utils/Web/UrlUtils.cs | 2 +- 12 files changed, 70 insertions(+), 74 deletions(-) diff --git a/Lib_CSharp b/Lib_CSharp index fa260546..6b7ed8c0 160000 --- a/Lib_CSharp +++ b/Lib_CSharp @@ -1 +1 @@ -Subproject commit fa26054678e1c938b1120f7eaab8b26476700245 +Subproject commit 6b7ed8c04f1728570c2f7575bc1ceab2b713f04b diff --git a/PoliNetwork.Graduatorie.Common/Enums/SchoolEnum.cs b/PoliNetwork.Graduatorie.Common/Enums/SchoolEnum.cs index 66b6eced..12bb8d69 100644 --- a/PoliNetwork.Graduatorie.Common/Enums/SchoolEnum.cs +++ b/PoliNetwork.Graduatorie.Common/Enums/SchoolEnum.cs @@ -28,7 +28,7 @@ public static string ToShortName(this SchoolEnum s) SchoolEnum.Design => "DES", SchoolEnum.Ingegneria => "ENG", SchoolEnum.Urbanistica => "URB", - _ => "UNK", + _ => "UNK" }; } } \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs index 599b87ca..525b664c 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs @@ -8,7 +8,6 @@ using PoliNetwork.Graduatorie.Parser.Objects.RankingNS; using PoliNetwork.Graduatorie.Parser.Objects.Tables.Course; using PoliNetwork.Graduatorie.Parser.Objects.Tables.Merit; -using PoliNetwork.Graduatorie.Parser.Utils; #endregion @@ -92,20 +91,20 @@ private static bool SameHashCourse(IReadOnlyCollection? aTableCours var aHash = aTableCourse.Select(variable => { - var hashWithoutLastUpdate = Hashing.GetHashFromListHash(variable.GetHashWithoutLastUpdate()); + var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); return hashWithoutLastUpdate; }).ToList(); var bHash = bTableCourse.Select(variable => { - var hashWithoutLastUpdate = Hashing.GetHashFromListHash(variable.GetHashWithoutLastUpdate()); + var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); return hashWithoutLastUpdate; }).ToList(); - var ai = Hashing.GetHashFromListHash(aHash); - var bi = Hashing.GetHashFromListHash(bHash); + var ai = aHash; + var bi = bHash; - return (ai ?? 0) == (bi ?? 0); + return ai == bi; } private static bool SameHashMerit(MeritTable? aTableMerit, MeritTable? bTableMerit) @@ -117,9 +116,7 @@ private static bool SameHashMerit(MeritTable? aTableMerit, MeritTable? bTableMer var ai = aTableMerit.GetHashWithoutLastUpdate(); var bi = bTableMerit.GetHashWithoutLastUpdate(); - var aii = Hashing.GetHashFromListHash(ai) ?? 0; - var bii = Hashing.GetHashFromListHash(bi) ?? 0; - return aii == bii; + return ai == bi; } private static Ranking? GetRankingFromFile(string path) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs index bdb25dc1..dcff36b8 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs @@ -12,31 +12,20 @@ namespace PoliNetwork.Graduatorie.Parser.Objects.Json; [Serializable] [JsonObject(MemberSerialization.Fields, NamingStrategyType = typeof(CamelCaseNamingStrategy))] -public class SingleCourseJson: IComparable +public class SingleCourseJson : IComparable { public string? BasePath; + public string? Id; public string? Link; public string? Location; - public string? Id; public RankingOrder? RankingOrder; public SchoolEnum? School; public int? Year; - public int GetHashWithoutLastUpdate() - { - var hashWithoutLastUpdate = Link?.GetHashCode() ?? "Link".GetHashCode(); - var hashCode = Id?.GetHashCode() ?? "Id".GetHashCode(); - var basePathInt = BasePath?.GetHashCode() ?? "BasePath".GetHashCode(); - var yearInt = Year?.GetHashCode() ?? "Year".GetHashCode(); - var schoolInt = School?.GetHashCode() ?? "School".GetHashCode(); - var code = "SingleCourseJson".GetHashCode(); - return hashWithoutLastUpdate ^ hashCode ^ basePathInt ^ yearInt ^ schoolInt ^ code; - } - public int CompareTo(SingleCourseJson? singleCourseJson) { if (singleCourseJson == null) return 1; - + if (Year != singleCourseJson.Year) return (Year ?? -1) < (singleCourseJson.Year ?? -1) ? -1 : 1; @@ -57,6 +46,17 @@ public int CompareTo(SingleCourseJson? singleCourseJson) return 0; } + public int GetHashWithoutLastUpdate() + { + var hashWithoutLastUpdate = Link?.GetHashCode() ?? "Link".GetHashCode(); + var hashCode = Id?.GetHashCode() ?? "Id".GetHashCode(); + var basePathInt = BasePath?.GetHashCode() ?? "BasePath".GetHashCode(); + var yearInt = Year?.GetHashCode() ?? "Year".GetHashCode(); + var schoolInt = School?.GetHashCode() ?? "School".GetHashCode(); + var code = "SingleCourseJson".GetHashCode(); + return hashWithoutLastUpdate ^ hashCode ^ basePathInt ^ yearInt ^ schoolInt ^ code; + } + public bool Is(CourseTable courseTable) { return (RankingOrder?.Phase ?? "") == courseTable.Title; diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs index 4a4fad20..ddea20f2 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs @@ -9,7 +9,6 @@ using PoliNetwork.Graduatorie.Parser.Objects.Json.Stats; using PoliNetwork.Graduatorie.Parser.Objects.Tables.Course; using PoliNetwork.Graduatorie.Parser.Objects.Tables.Merit; -using PoliNetwork.Graduatorie.Parser.Utils; using PoliNetwork.Graduatorie.Parser.Utils.Output; #endregion @@ -30,19 +29,19 @@ public class Ranking : IComparable public RankingUrl? Url; public int? Year; - public RankingSummaryStudent GetRankingSummaryStudent() - { - return new RankingSummaryStudent(RankingOrder?.Phase, School, Year, Url); - } - public int CompareTo(Ranking? other) { if (ReferenceEquals(this, other)) return 0; if (ReferenceEquals(null, other)) return 1; - + return string.Compare(GetId(), other.GetId(), StringComparison.Ordinal); } + public RankingSummaryStudent GetRankingSummaryStudent() + { + return new RankingSummaryStudent(RankingOrder?.Phase, School, Year, Url); + } + /*** * Ottieni l'hash senza considerare il valore di LastUpdate @@ -57,7 +56,7 @@ public int GetHashWithoutLastUpdate() i ^= Url?.GetHashWithoutLastUpdate() ?? "Url".GetHashCode(); i ^= Year?.GetHashCode() ?? "Year".GetHashCode(); var iMerit = ByMerit?.GetHashWithoutLastUpdate(); - i ^= Hashing.GetHashFromListHash(iMerit) ?? "ByMerit".GetHashCode(); + i ^= iMerit ?? "ByMerit".GetHashCode(); if (ByCourse == null) @@ -66,7 +65,7 @@ public int GetHashWithoutLastUpdate() i = ByCourse.Aggregate(i, (current, variable) => { var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); - var iList = Hashing.GetHashFromListHash(hashWithoutLastUpdate) ?? "empty".GetHashCode(); + var iList = hashWithoutLastUpdate; return current ^ iList; }); @@ -113,7 +112,7 @@ public string GetFilename() public string GetId() { var idList = new List(); - + var schoolShort = School?.ToShortName(); if (schoolShort != null) idList.Add(schoolShort); @@ -122,7 +121,7 @@ public string GetId() var orderId = RankingOrder?.GetId(); if (orderId != null) idList.Add(orderId); - + var fallback = DateTime.UtcNow.ToString("yyyyMMddTHHmmss", CultureInfo.InvariantCulture) + "Z"; if (idList.Count == 0) idList.Add(fallback); diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingOrder.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingOrder.cs index e7407401..561908ca 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingOrder.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/RankingOrder.cs @@ -11,6 +11,9 @@ namespace PoliNetwork.Graduatorie.Parser.Objects.RankingNS; [JsonObject(MemberSerialization.Fields, NamingStrategyType = typeof(CamelCaseNamingStrategy))] public class RankingOrder { + public bool IsAnticipata; // used for DES/URB rankings until 2023 + public bool IsEnglish; + public bool IsExtraEu; public string? Phase; // the original string (e.g. " //esempio: @@ -18,15 +21,12 @@ public class RankingOrder //prima graduatoria di seconda fase:{primary:2, secondary:1} public int? Primary; public int? Secondary; - public bool IsAnticipata; // used for DES/URB rankings until 2023 - public bool IsExtraEu; - public bool IsEnglish; public RankingOrder(string phase, bool isExtraEu = false, bool isEnglish = false) { Phase = phase; ParsePhaseString(phase); - + IsExtraEu = isExtraEu; IsEnglish = isEnglish; } @@ -35,12 +35,12 @@ private void ParsePhaseString(string phase) { var s = phase.ToUpper().Trim(); if (string.IsNullOrEmpty(s)) return; - + var strings = s.Split(" "); - + IsAnticipata = s.Contains("ANTICIPATA"); if (IsAnticipata) return; - + Primary = ExtractPhaseNumberByKey(strings, "FASE"); Secondary = ExtractPhaseNumberByKey(strings, "GRADUATORIA"); } @@ -74,22 +74,19 @@ private void ParsePhaseString(string phase) public string GetId() { var idList = new List(); - if (IsAnticipata) idList.Add($"anticipata"); + if (IsAnticipata) idList.Add("anticipata"); if (Primary != null) idList.Add($"{Primary}fase"); if (Secondary != null) idList.Add($"{Secondary}grad"); - + var cleanPhase = Phase?.Replace("_", "").Replace("-", "").Replace(" ", "_").ToLower() ?? ""; - var noOrder = IsAnticipata == false && Primary == null && Secondary == null; + var noOrder = IsAnticipata == false && Primary == null && Secondary == null; var isSingleExtraEu = noOrder && cleanPhase.Contains("extraue"); - if (noOrder) - { - idList.Add(isSingleExtraEu ? "extraeu" : cleanPhase); - } - + if (noOrder) idList.Add(isSingleExtraEu ? "extraeu" : cleanPhase); + idList.Add(IsEnglish ? "eng" : "ita"); if (IsExtraEu && !isSingleExtraEu) idList.Add("extraeu"); // the second condition is to avoid double extraeu - + var id = string.Join("_", idList); return id; } @@ -111,4 +108,4 @@ public void Merge(RankingOrder? rankingRankingOrder) Primary ??= rankingRankingOrder?.Primary; Secondary ??= rankingRankingOrder?.Secondary; } -} +} \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Parser/Objects/StudentResult.cs b/PoliNetwork.Graduatorie.Parser/Objects/StudentResult.cs index 267e800d..37d69a09 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/StudentResult.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/StudentResult.cs @@ -3,6 +3,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Serialization; using PoliNetwork.Graduatorie.Common.Objects; +using PoliNetwork.Graduatorie.Parser.Utils; #endregion @@ -22,7 +23,7 @@ public class StudentResult public decimal? Result; public SortedDictionary? SectionsResults; - public List GetHashWithoutLastUpdate() + public int GetHashWithoutLastUpdate() { var r = new List { @@ -47,6 +48,6 @@ public class StudentResult r.Add(SectionsResults.Aggregate("SectionsResultsFull".GetHashCode(), (current, variable) => current ^ variable.Key.GetHashCode() ^ variable.Value.GetHashCode())); - return r; + return Hashing.GetHashFromListHash(r); } } \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Tables/Merit/MeritTable.cs b/PoliNetwork.Graduatorie.Parser/Objects/Tables/Merit/MeritTable.cs index a796b4c5..1add9a7d 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Tables/Merit/MeritTable.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Tables/Merit/MeritTable.cs @@ -17,7 +17,7 @@ public class MeritTable public List? Rows; public int? Year; - public List GetHashWithoutLastUpdate() + public int GetHashWithoutLastUpdate() { var r = new List { "MeritTable".GetHashCode() }; if (Headers != null) @@ -30,14 +30,13 @@ public class MeritTable r.Add(Rows.Aggregate("RowsFull".GetHashCode(), (current, variable) => { var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); - var hashFromListHash = Hashing.GetHashFromListHash(hashWithoutLastUpdate) ?? "empty3".GetHashCode(); - return current ^ hashFromListHash; + return current ^ hashWithoutLastUpdate; })); else r.Add("RowsEmpty".GetHashCode()); r.Add(Year?.GetHashCode() ?? "Year".GetHashCode()); r.Add(Path?.GetHashCode() ?? "Path".GetHashCode()); - return r; + return Hashing.GetHashFromListHash(r); } } \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Hashing.cs b/PoliNetwork.Graduatorie.Parser/Utils/Hashing.cs index e71a7ceb..b5eb024f 100644 --- a/PoliNetwork.Graduatorie.Parser/Utils/Hashing.cs +++ b/PoliNetwork.Graduatorie.Parser/Utils/Hashing.cs @@ -2,13 +2,14 @@ namespace PoliNetwork.Graduatorie.Parser.Utils; public static class Hashing { - public static int? GetHashFromListHash(IReadOnlyCollection? iMerit) + public static int GetHashFromListHash(IReadOnlyCollection? iMerit) { if (iMerit == null) - return null; + return 0; if (iMerit.Count == 0) - return null; + return 0; - return iMerit.Aggregate(0, (current, variable) => current ^ variable ?? 0); + var hashFromListHash = iMerit.Aggregate(0, (current, variable) => current ^ variable ?? 0); + return hashFromListHash; } } \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Transformer/ParserNS/Parser.cs b/PoliNetwork.Graduatorie.Parser/Utils/Transformer/ParserNS/Parser.cs index 781c9e69..106e9500 100644 --- a/PoliNetwork.Graduatorie.Parser/Utils/Transformer/ParserNS/Parser.cs +++ b/PoliNetwork.Graduatorie.Parser/Utils/Transformer/ParserNS/Parser.cs @@ -208,26 +208,27 @@ private RankingsSet ParseNewRankings(IReadOnlyCollection htmls) var extraEuStr = intestazioni[4].Split("\n")[0].ToLower(); var isExtraEu = extraEuStr.Contains("extra-ue"); - if (ranking.Year < 2024) { + if (ranking.Year < 2024) + { // layout valid until 2023 var phase = string.Join(" ", intestazioni[3].Split(" - ")[1..]); ranking.RankingOrder = new RankingOrder(phase, isExtraEu); if (ranking.School == SchoolEnum.Architettura && ranking.RankingOrder.Primary == null && ranking.RankingOrder.Secondary == null && ranking.RankingOrder.IsExtraEu) - { // this is a fallback for 2020-2023: // POLIMI was used to add the ranking number (Secondary, e.g. "Prima Graduatoria") for ExtraEU starting // from the second ranking. // e.g. Extra-EU first ranking => phase = "Extra-ue", // Extra-EU second ranking => phase = "Extra-ue - Seconda Graduatoria" // so this is a fallback to add the equivalent of "Prima Graduatoria" to the first ExtraEU ranking. - ranking.RankingOrder.Secondary = 1; - } - } else { + } + else + { // layout valid since 2024 (if the layout changes again, make another else if) var phase = intestazioni[3]; - var isEnglish = intestazioni[2].Contains("taught in english") || intestazioni[2].Contains("erogati in inglese"); + var isEnglish = intestazioni[2].Contains("taught in english") || + intestazioni[2].Contains("erogati in inglese"); ranking.RankingOrder = new RankingOrder(phase, isExtraEu, isEnglish); } @@ -774,4 +775,4 @@ private IEnumerable ParseLocalHtmlFiles() ); return obj2; } -} +} \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Scraper/Utils/Web/Scraper.cs b/PoliNetwork.Graduatorie.Scraper/Utils/Web/Scraper.cs index 2dc91e58..f13f0154 100644 --- a/PoliNetwork.Graduatorie.Scraper/Utils/Web/Scraper.cs +++ b/PoliNetwork.Graduatorie.Scraper/Utils/Web/Scraper.cs @@ -40,7 +40,8 @@ private IEnumerable ScrapeAvvisiFuturiStudenti() var page = _web.Load(AvvisiFuturiStudentiUrl).DocumentNode; var newsCards = - page.SelectNodes("//div[contains(@class, 'news')]//div[contains(@class, 'row--card')]//div[contains(@class, 'card__content')]"); + page.SelectNodes( + "//div[contains(@class, 'news')]//div[contains(@class, 'row--card')]//div[contains(@class, 'card__content')]"); if (newsCards == null) return links; foreach (var card in newsCards) @@ -48,7 +49,7 @@ private IEnumerable ScrapeAvvisiFuturiStudenti() var title = card.Descendants("h5").First(); var titleValid = title != null && IsValidText(title.InnerText); - var body = card.Descendants("p").Where(el => el.ParentNode.HasClass("news-bodytext")).First(); + var body = card.Descendants("p").First(el => el.ParentNode.HasClass("news-bodytext")); var bodyValid = body != null && IsValidText(body.InnerText); var aTag = card.Descendants("a").First(); @@ -134,4 +135,4 @@ private bool IsValidText(string text) return null; } } -} +} \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Scraper/Utils/Web/UrlUtils.cs b/PoliNetwork.Graduatorie.Scraper/Utils/Web/UrlUtils.cs index fcf0d953..30bffc81 100644 --- a/PoliNetwork.Graduatorie.Scraper/Utils/Web/UrlUtils.cs +++ b/PoliNetwork.Graduatorie.Scraper/Utils/Web/UrlUtils.cs @@ -40,4 +40,4 @@ public static bool CheckUrl(RankingUrl? url) return false; } } -} +} \ No newline at end of file From e0bb733da31e415339978e06f14db715003dedec Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sat, 6 Jul 2024 10:58:25 +0200 Subject: [PATCH 2/6] Update Lib_CSharp --- Lib_CSharp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib_CSharp b/Lib_CSharp index fa260546..6b7ed8c0 160000 --- a/Lib_CSharp +++ b/Lib_CSharp @@ -1 +1 @@ -Subproject commit fa26054678e1c938b1120f7eaab8b26476700245 +Subproject commit 6b7ed8c04f1728570c2f7575bc1ceab2b713f04b From 0b76585dbc1ea0aabb48a249f611c72ae609189d Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sat, 6 Jul 2024 11:03:10 +0200 Subject: [PATCH 3/6] merge conflicts --- .../Objects/Json/Indexes/IndexJsonBase.cs | 27 +++++------- .../Objects/Json/SingleCourseJson.cs | 16 ++------ .../Objects/RankingNS/Ranking.cs | 41 ++++--------------- 3 files changed, 21 insertions(+), 63 deletions(-) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs index 5d4a8d3d..7cdab8e2 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs @@ -6,11 +6,9 @@ using PoliNetwork.Graduatorie.Common.Objects; using PoliNetwork.Graduatorie.Parser.Objects.Json.Indexes.Specific; using PoliNetwork.Graduatorie.Parser.Objects.RankingNS; -<<<<<<< HEAD -======= + using PoliNetwork.Graduatorie.Parser.Objects.Tables.Course; using PoliNetwork.Graduatorie.Parser.Objects.Tables.Merit; ->>>>>>> main #endregion @@ -28,11 +26,18 @@ public void WriteToFile(string outFolder, string pathFile) var mainJsonString = JsonConvert.SerializeObject(this, Culture.JsonSerializerSettings); File.WriteAllText(mainJsonPath, mainJsonString); } -<<<<<<< HEAD - + + public static void WriteAllIndexes(RankingsSet rankingsSet, string outFolder, ArgsConfig argsConfig) { -======= + //let's write all single json files + WriteSingleJsons(rankingsSet, outFolder, argsConfig); + //now let's write each single different index + BySchoolYearJson.From(rankingsSet)?.WriteToFile(outFolder, BySchoolYearJson.PathCustom); + ByYearSchoolJson.From(rankingsSet)?.WriteToFile(outFolder, ByYearSchoolJson.PathCustom); + BySchoolYearCourseJson.From(rankingsSet)?.WriteToFile(outFolder, BySchoolYearCourseJson.PathCustom); + } + public static void WriteSingleJsons(RankingsSet? set, string outFolder, ArgsConfig argsConfig) @@ -135,15 +140,5 @@ private static bool SameHashMerit(MeritTable? aTableMerit, MeritTable? bTableMer return j; } - public static void IndexesWrite(RankingsSet? rankingsSet, string outFolder, ArgsConfig argsConfig) - { - //let's write all single json files - WriteSingleJsons(rankingsSet, outFolder, argsConfig); ->>>>>>> main - //now let's write each single different index - BySchoolYearJson.From(rankingsSet)?.WriteToFile(outFolder, BySchoolYearJson.PathCustom); - ByYearSchoolJson.From(rankingsSet)?.WriteToFile(outFolder, ByYearSchoolJson.PathCustom); - BySchoolYearCourseJson.From(rankingsSet)?.WriteToFile(outFolder, BySchoolYearCourseJson.PathCustom); - } } \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs index 360e4c49..3fdaf4b6 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs @@ -22,7 +22,7 @@ public class SingleCourseJson : IComparable public SchoolEnum? School; public int? Year; -<<<<<<< HEAD + public static SingleCourseJson From(Ranking ranking, CourseTable? course) { var basePath = $"{ranking.School}/{ranking.Year}/"; // "Ingegneria/2023" @@ -49,8 +49,7 @@ public int GetHashWithoutLastUpdate() return hashWithoutLastUpdate ^ hashCode ^ basePathInt ^ yearInt ^ schoolInt ^ code; } -======= ->>>>>>> main + public int CompareTo(SingleCourseJson? singleCourseJson) { if (singleCourseJson == null) return 1; @@ -75,16 +74,7 @@ public int CompareTo(SingleCourseJson? singleCourseJson) return 0; } - public int GetHashWithoutLastUpdate() - { - var hashWithoutLastUpdate = Link?.GetHashCode() ?? "Link".GetHashCode(); - var hashCode = Id?.GetHashCode() ?? "Id".GetHashCode(); - var basePathInt = BasePath?.GetHashCode() ?? "BasePath".GetHashCode(); - var yearInt = Year?.GetHashCode() ?? "Year".GetHashCode(); - var schoolInt = School?.GetHashCode() ?? "School".GetHashCode(); - var code = "SingleCourseJson".GetHashCode(); - return hashWithoutLastUpdate ^ hashCode ^ basePathInt ^ yearInt ^ schoolInt ^ code; - } + public bool Is(CourseTable courseTable) { diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs index de907a59..89128e8a 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs @@ -30,12 +30,8 @@ public class Ranking : IComparable, IEquatable public RankingUrl? Url; public int? Year; -<<<<<<< HEAD - public RankingSummaryStudent GetRankingSummaryStudent() - { - return new RankingSummaryStudent(RankingOrder?.Phase, School, Year, Url); - } - + + public static Ranking? FromJson(string fullPath) { if (!File.Exists(fullPath)) return null; @@ -45,8 +41,7 @@ public RankingSummaryStudent GetRankingSummaryStudent() return ranking; } -======= ->>>>>>> main + public int CompareTo(Ranking? other) { if (ReferenceEquals(this, other)) return 0; @@ -59,37 +54,15 @@ public RankingSummaryStudent GetRankingSummaryStudent() { return new RankingSummaryStudent(RankingOrder?.Phase, School, Year, Url); } + public bool Equals(Ranking? other) { -<<<<<<< HEAD + if (other == null) return false; return GetHashWithoutLastUpdate() == other.GetHashWithoutLastUpdate(); -======= - var i = "Ranking".GetHashCode(); - i ^= Extra?.GetHashCode() ?? "Extra".GetHashCode(); - i ^= RankingOrder?.GetHashWithoutLastUpdate() ?? "RankingOrder".GetHashCode(); - i ^= RankingSummary?.GetHashWithoutLastUpdate() ?? "RankingSummary".GetHashCode(); - i ^= School?.GetHashCode() ?? "School".GetHashCode(); - i ^= Url?.GetHashWithoutLastUpdate() ?? "Url".GetHashCode(); - i ^= Year?.GetHashCode() ?? "Year".GetHashCode(); - var iMerit = ByMerit?.GetHashWithoutLastUpdate(); - i ^= iMerit ?? "ByMerit".GetHashCode(); - - - if (ByCourse == null) - i ^= "ByCourse".GetHashCode(); - else - i = ByCourse.Aggregate(i, (current, variable) => - { - var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); - var iList = hashWithoutLastUpdate; - return current ^ iList; - }); - return i; ->>>>>>> main } @@ -217,7 +190,7 @@ public int GetHashWithoutLastUpdate() i ^= Url?.GetHashWithoutLastUpdate() ?? "Url".GetHashCode(); i ^= Year?.GetHashCode() ?? "Year".GetHashCode(); var iMerit = ByMerit?.GetHashWithoutLastUpdate(); - i ^= Hashing.GetHashFromListHash(iMerit) ?? "ByMerit".GetHashCode(); + i ^= (iMerit) ?? "ByMerit".GetHashCode(); if (ByCourse == null) @@ -226,7 +199,7 @@ public int GetHashWithoutLastUpdate() i = ByCourse.Aggregate(i, (current, variable) => { var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); - var iList = Hashing.GetHashFromListHash(hashWithoutLastUpdate) ?? "empty".GetHashCode(); + var iList = (hashWithoutLastUpdate); return current ^ iList; }); From 3baedee5bc4e4875a6da41d2c15869b8539d46b2 Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sat, 6 Jul 2024 11:04:15 +0200 Subject: [PATCH 4/6] Update Ranking.cs --- .../Objects/RankingNS/Ranking.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs index 89128e8a..0f9f7eb3 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs @@ -64,6 +64,16 @@ public bool Equals(Ranking? other) return GetHashWithoutLastUpdate() == other.GetHashWithoutLastUpdate(); } + + public override bool Equals(object? obj) + { + return Equals(obj as Ranking); + } + + public override int GetHashCode() + { + return this.GetHashWithoutLastUpdate(); + } public bool IsSimilarTo(Ranking ranking) @@ -205,4 +215,6 @@ public int GetHashWithoutLastUpdate() return i; } + + } \ No newline at end of file From 2d786740f9eeb5888df7ce3f2c1d96f3010b7492 Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sat, 6 Jul 2024 11:04:37 +0200 Subject: [PATCH 5/6] Update EnrollType.cs, IndexJsonBase.cs, BySchoolYearCourseJson.cs, and 4 more files --- .../Objects/EnrollType.cs | 2 +- .../Objects/Json/Indexes/IndexJsonBase.cs | 4 -- .../Specific/BySchoolYearCourseJson.cs | 2 + .../Objects/Json/SingleCourseJson.cs | 51 +++++++++---------- .../Objects/RankingNS/Ranking.cs | 44 +++++++--------- .../Objects/Tables/Course/CourseTable.cs | 4 +- .../Utils/Output/OutputWriteUtil.cs | 2 +- 7 files changed, 50 insertions(+), 59 deletions(-) diff --git a/PoliNetwork.Graduatorie.Common/Objects/EnrollType.cs b/PoliNetwork.Graduatorie.Common/Objects/EnrollType.cs index 2fb946ec..09d44d99 100644 --- a/PoliNetwork.Graduatorie.Common/Objects/EnrollType.cs +++ b/PoliNetwork.Graduatorie.Common/Objects/EnrollType.cs @@ -23,7 +23,7 @@ public int GetHashWithoutLastUpdate() return i; } - + public static EnrollType From(string? rowCanEnrollInto, bool rowCanEnroll) { if (rowCanEnroll == false) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs index 7cdab8e2..e7872e95 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs @@ -6,7 +6,6 @@ using PoliNetwork.Graduatorie.Common.Objects; using PoliNetwork.Graduatorie.Parser.Objects.Json.Indexes.Specific; using PoliNetwork.Graduatorie.Parser.Objects.RankingNS; - using PoliNetwork.Graduatorie.Parser.Objects.Tables.Course; using PoliNetwork.Graduatorie.Parser.Objects.Tables.Merit; @@ -39,7 +38,6 @@ public static void WriteAllIndexes(RankingsSet rankingsSet, string outFolder, Ar } - public static void WriteSingleJsons(RankingsSet? set, string outFolder, ArgsConfig argsConfig) { if (set == null) @@ -139,6 +137,4 @@ private static bool SameHashMerit(MeritTable? aTableMerit, MeritTable? bTableMer var j = JsonConvert.DeserializeObject(x, Culture.JsonSerializerSettings); return j; } - - } \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs index ae84a487..e4e9c9e0 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/Specific/BySchoolYearCourseJson.cs @@ -1,9 +1,11 @@ #region + using System.Diagnostics; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; using PoliNetwork.Graduatorie.Common.Enums; using PoliNetwork.Graduatorie.Parser.Objects.RankingNS; + #endregion namespace PoliNetwork.Graduatorie.Parser.Objects.Json.Indexes.Specific; diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs index 3fdaf4b6..9f5dbb0b 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/SingleCourseJson.cs @@ -23,6 +23,31 @@ public class SingleCourseJson : IComparable public int? Year; + public int CompareTo(SingleCourseJson? singleCourseJson) + { + if (singleCourseJson == null) return 1; + + if (Year != singleCourseJson.Year) + return (Year ?? -1) < (singleCourseJson.Year ?? -1) ? -1 : 1; + + if (School != singleCourseJson.School) return School < singleCourseJson.School ? -1 : 1; + + if (BasePath != singleCourseJson.BasePath) + return string.Compare(BasePath ?? "", singleCourseJson.BasePath ?? "", StringComparison.InvariantCulture); + + if (Link != singleCourseJson.Link) + return string.Compare(Link ?? "", singleCourseJson.Link ?? "", StringComparison.InvariantCulture); + + if (Location != singleCourseJson.Location) + return string.Compare(Location ?? "", singleCourseJson.Location ?? "", StringComparison.InvariantCulture); + + if (Id != singleCourseJson.Id) + return string.Compare(Id ?? "", singleCourseJson.Id ?? "", StringComparison.InvariantCulture); + + return 0; + } + + public static SingleCourseJson From(Ranking ranking, CourseTable? course) { var basePath = $"{ranking.School}/{ranking.Year}/"; // "Ingegneria/2023" @@ -50,32 +75,6 @@ public int GetHashWithoutLastUpdate() } - public int CompareTo(SingleCourseJson? singleCourseJson) - { - if (singleCourseJson == null) return 1; - - if (Year != singleCourseJson.Year) - return (Year ?? -1) < (singleCourseJson.Year ?? -1) ? -1 : 1; - - if (School != singleCourseJson.School) return School < singleCourseJson.School ? -1 : 1; - - if (BasePath != singleCourseJson.BasePath) - return string.Compare(BasePath ?? "", singleCourseJson.BasePath ?? "", StringComparison.InvariantCulture); - - if (Link != singleCourseJson.Link) - return string.Compare(Link ?? "", singleCourseJson.Link ?? "", StringComparison.InvariantCulture); - - if (Location != singleCourseJson.Location) - return string.Compare(Location ?? "", singleCourseJson.Location ?? "", StringComparison.InvariantCulture); - - if (Id != singleCourseJson.Id) - return string.Compare(Id ?? "", singleCourseJson.Id ?? "", StringComparison.InvariantCulture); - - return 0; - } - - - public bool Is(CourseTable courseTable) { return (RankingOrder?.Phase ?? "") == courseTable.Title; diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs index 0f9f7eb3..8880c66b 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs @@ -31,17 +31,6 @@ public class Ranking : IComparable, IEquatable public int? Year; - - public static Ranking? FromJson(string fullPath) - { - if (!File.Exists(fullPath)) return null; - - var str = File.ReadAllText(fullPath); - var ranking = JsonConvert.DeserializeObject(str, Culture.JsonSerializerSettings); - return ranking; - } - - public int CompareTo(Ranking? other) { if (ReferenceEquals(this, other)) return 0; @@ -50,21 +39,28 @@ public int CompareTo(Ranking? other) return string.Compare(GetId(), other.GetId(), StringComparison.Ordinal); } - public RankingSummaryStudent GetRankingSummaryStudent() + + public bool Equals(Ranking? other) { - return new RankingSummaryStudent(RankingOrder?.Phase, School, Year, Url); + if (other == null) return false; + return GetHashWithoutLastUpdate() == other.GetHashWithoutLastUpdate(); } - - public bool Equals(Ranking? other) + public static Ranking? FromJson(string fullPath) { + if (!File.Exists(fullPath)) return null; - if (other == null) return false; - return GetHashWithoutLastUpdate() == other.GetHashWithoutLastUpdate(); + var str = File.ReadAllText(fullPath); + var ranking = JsonConvert.DeserializeObject(str, Culture.JsonSerializerSettings); + return ranking; + } + public RankingSummaryStudent GetRankingSummaryStudent() + { + return new RankingSummaryStudent(RankingOrder?.Phase, School, Year, Url); } - + public override bool Equals(object? obj) { return Equals(obj as Ranking); @@ -72,7 +68,7 @@ public override bool Equals(object? obj) public override int GetHashCode() { - return this.GetHashWithoutLastUpdate(); + return GetHashWithoutLastUpdate(); } @@ -174,7 +170,7 @@ public void WriteAsJson(string outFolder, bool forceReparse = false) { var folderPath = GetBasePath(outFolder); Directory.CreateDirectory(folderPath); - + var fullPath = GetFullPath(outFolder); var savedRanking = FromJson(fullPath); @@ -186,7 +182,7 @@ public void WriteAsJson(string outFolder, bool forceReparse = false) File.WriteAllText(fullPath, rankingJsonString); } } - + /*** * Ottieni l'hash senza considerare il valore di LastUpdate */ @@ -200,7 +196,7 @@ public int GetHashWithoutLastUpdate() i ^= Url?.GetHashWithoutLastUpdate() ?? "Url".GetHashCode(); i ^= Year?.GetHashCode() ?? "Year".GetHashCode(); var iMerit = ByMerit?.GetHashWithoutLastUpdate(); - i ^= (iMerit) ?? "ByMerit".GetHashCode(); + i ^= iMerit ?? "ByMerit".GetHashCode(); if (ByCourse == null) @@ -209,12 +205,10 @@ public int GetHashWithoutLastUpdate() i = ByCourse.Aggregate(i, (current, variable) => { var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); - var iList = (hashWithoutLastUpdate); + var iList = hashWithoutLastUpdate; return current ^ iList; }); return i; } - - } \ No newline at end of file diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs b/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs index 17aac09b..791c4b5e 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Tables/Course/CourseTable.cs @@ -31,8 +31,8 @@ public RankingSummaryStudent GetRankingSummaryStudent(Ranking ranking) } /// - /// Get the course location if present, otherwise get the placeholder (constant). - /// Useful for index purposes. + /// Get the course location if present, otherwise get the placeholder (constant). + /// Useful for index purposes. /// /// A string with the location or the placeholder public string GetFixedLocation() diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Output/OutputWriteUtil.cs b/PoliNetwork.Graduatorie.Parser/Utils/Output/OutputWriteUtil.cs index f8dfad6f..f603a5c3 100644 --- a/PoliNetwork.Graduatorie.Parser/Utils/Output/OutputWriteUtil.cs +++ b/PoliNetwork.Graduatorie.Parser/Utils/Output/OutputWriteUtil.cs @@ -23,7 +23,7 @@ public OutputWriteUtil(ArgsConfig argsConfig) public void SaveOutputs(RankingsSet rankingsSet, DateFound dateFound) { var outFolder = Path.Join(_config.DataFolder, Constants.OutputFolder); - + rankingsSet.WriteAllRankings(outFolder, _config.ForceReparsing); IndexJsonBase.WriteAllIndexes(rankingsSet, outFolder, _config); StatsJson.Write(outFolder, rankingsSet, _config); From 6a63b03a7e7eae1e45a97e7d41a047268c1606e4 Mon Sep 17 00:00:00 2001 From: userbot_github Date: Sat, 6 Jul 2024 11:07:03 +0200 Subject: [PATCH 6/6] Update IndexJsonBase.cs, Ranking.cs, and Hashing.cs --- .../Objects/Json/Indexes/IndexJsonBase.cs | 12 ++++++------ .../Objects/RankingNS/Ranking.cs | 3 +-- PoliNetwork.Graduatorie.Parser/Utils/Hashing.cs | 9 +++++++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs index e7872e95..15124454 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/Json/Indexes/IndexJsonBase.cs @@ -8,6 +8,7 @@ using PoliNetwork.Graduatorie.Parser.Objects.RankingNS; using PoliNetwork.Graduatorie.Parser.Objects.Tables.Course; using PoliNetwork.Graduatorie.Parser.Objects.Tables.Merit; +using PoliNetwork.Graduatorie.Parser.Utils; #endregion @@ -100,22 +101,21 @@ private static bool SameHashCourse(IReadOnlyCollection? aTableCours if (aTableCourse.Count != bTableCourse.Count) return false; - var aHash = aTableCourse.Select(variable => + var aList = aTableCourse.Select(variable => { var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); return hashWithoutLastUpdate; }).ToList(); + var aHash = Hashing.GetHashFromListHash(aList); - var bHash = bTableCourse.Select(variable => + var bList = bTableCourse.Select(variable => { var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); return hashWithoutLastUpdate; }).ToList(); + var bHash = Hashing.GetHashFromListHash(bList); - var ai = aHash; - var bi = bHash; - - return ai == bi; + return aHash == bHash; } private static bool SameHashMerit(MeritTable? aTableMerit, MeritTable? bTableMerit) diff --git a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs index 8880c66b..8ee534c0 100644 --- a/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs +++ b/PoliNetwork.Graduatorie.Parser/Objects/RankingNS/Ranking.cs @@ -205,8 +205,7 @@ public int GetHashWithoutLastUpdate() i = ByCourse.Aggregate(i, (current, variable) => { var hashWithoutLastUpdate = variable.GetHashWithoutLastUpdate(); - var iList = hashWithoutLastUpdate; - return current ^ iList; + return current ^ hashWithoutLastUpdate; }); return i; diff --git a/PoliNetwork.Graduatorie.Parser/Utils/Hashing.cs b/PoliNetwork.Graduatorie.Parser/Utils/Hashing.cs index b5eb024f..9d435101 100644 --- a/PoliNetwork.Graduatorie.Parser/Utils/Hashing.cs +++ b/PoliNetwork.Graduatorie.Parser/Utils/Hashing.cs @@ -12,4 +12,13 @@ public static int GetHashFromListHash(IReadOnlyCollection? iMerit) var hashFromListHash = iMerit.Aggregate(0, (current, variable) => current ^ variable ?? 0); return hashFromListHash; } + + public static int GetHashFromListHash(List iMerit) + { + if (iMerit.Count == 0) + return 0; + + var hashFromListHash = iMerit.Aggregate(0, (current, variable) => current ^ variable); + return hashFromListHash; + } } \ No newline at end of file