diff --git a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java index ae9b41fd..c3c31684 100644 --- a/src/main/java/org/mtransit/parser/DefaultAgencyTools.java +++ b/src/main/java/org/mtransit/parser/DefaultAgencyTools.java @@ -318,6 +318,34 @@ public String getAgencyColor(@NotNull GAgency gAgency, @NotNull GSpec gSpec) { return getAgencyColor(); } + @Nullable + @Override + public String getAgencyIdCleanupRegex() { + if (Configs.getAgencyConfig() != null) { + return Configs.getAgencyConfig().getAgencyIdCleanupRegex(); + } + return null; // OPT-IN feature + } + + @Nullable + private Pattern agencyIdCleanupPattern = null; + + private boolean agencyIdCleanupPatternSet = false; + + private @Nullable Pattern getAgencyIdCleanupPattern() { + if (this.agencyIdCleanupPattern == null && !agencyIdCleanupPatternSet) { + this.agencyIdCleanupPattern = GTFSCommons.makeIdCleanupPattern(getAgencyIdCleanupRegex()); + this.agencyIdCleanupPatternSet = true; + } + return this.agencyIdCleanupPattern; + } + + @NotNull + @Override + public String cleanAgencyId(@NotNull String gAgencyId) { + return GTFSCommons.cleanOriginalString(gAgencyId, getAgencyIdCleanupPattern()); + } + @Nullable @Override public String getAgencyId() { diff --git a/src/main/java/org/mtransit/parser/config/gtfs/data/AgencyConfig.kt b/src/main/java/org/mtransit/parser/config/gtfs/data/AgencyConfig.kt index a3423d9c..81acfc50 100644 --- a/src/main/java/org/mtransit/parser/config/gtfs/data/AgencyConfig.kt +++ b/src/main/java/org/mtransit/parser/config/gtfs/data/AgencyConfig.kt @@ -15,6 +15,8 @@ data class AgencyConfig( */ @SerialName("other_agency_ids") val otherAgencyIds: List = emptyList(), // OPT-IN filter + @SerialName("agency_id_cleanup_regex") + val agencyIdCleanupRegex: String? = null, // optional /** * Route type filter (integer from GTFS Static `routes.txt` > `route_type` field) * (useful when multiple route type in same GTFS) diff --git a/src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java b/src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java index 4774fda8..3859ab59 100644 --- a/src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java +++ b/src/main/java/org/mtransit/parser/gtfs/GAgencyTools.java @@ -73,6 +73,12 @@ public interface GAgencyTools { @Nullable String getAgencyId(); + @Nullable + String getAgencyIdCleanupRegex(); + + @NotNull + String cleanAgencyId(@NotNull String gAgencyId); + @NotNull Integer getAgencyRouteType(); diff --git a/src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java b/src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java index dcdbf87d..d7406182 100644 --- a/src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java +++ b/src/main/java/org/mtransit/parser/mt/GenerateMObjectsTask.java @@ -589,23 +589,16 @@ private void parseGTrips(HashMap mSchedules, && directionStopTimesHeadsign != null && !directionStopTimesHeadsign.isEmpty()) { if (mDirectionStopTimesHeadsign.containsKey(mDirection.getId())) { - if (!mDirectionStopTimesHeadsign.get(mDirection.getId()).equals(directionStopTimesHeadsign)) { - if (!mDirectionStopTimesHeadsign.get(mDirection.getId()).contains(directionStopTimesHeadsign)) { + final String cDirectionStopTimesHeadsign = mDirectionStopTimesHeadsign.get(mDirection.getId()); + if (!cDirectionStopTimesHeadsign.equals(directionStopTimesHeadsign)) { + if (!cDirectionStopTimesHeadsign.contains(directionStopTimesHeadsign)) { MTLog.log("%s: Trip stop times head-sign different for same direction ID ('%s'!='%s')", this.routeId, - directionStopTimesHeadsign, mDirectionStopTimesHeadsign.get(mDirection.getId())); + directionStopTimesHeadsign, cDirectionStopTimesHeadsign); } - String mergedHeadsignValue = MDirection.mergeHeadsignValue( - mDirectionStopTimesHeadsign.get(mDirection.getId()), + final String mergedHeadsignValue = MDirection.mergeHeadsignValue( + cDirectionStopTimesHeadsign, directionStopTimesHeadsign ); - if (mergedHeadsignValue != null) { - mergedHeadsignValue = agencyTools.cleanDirectionHeadsign( - gRoute, - gTrip.getDirectionIdOrDefault(), - true, - mergedHeadsignValue - ); - } mDirectionStopTimesHeadsign.put(mDirection.getId(), mergedHeadsignValue); } } else { diff --git a/src/main/java/org/mtransit/parser/mt/MGenerator.java b/src/main/java/org/mtransit/parser/mt/MGenerator.java index 2836b3e0..fc2fd435 100644 --- a/src/main/java/org/mtransit/parser/mt/MGenerator.java +++ b/src/main/java/org/mtransit/parser/mt/MGenerator.java @@ -315,10 +315,10 @@ public static void dumpFiles(@NotNull GAgencyTools gAgencyTools, // STRINGS dumpStrings(mSpec, fileBase, deleteAll, dataDirF, rawDirF, dbConnection); // AFTER ALL OTHER TABLE W/ STRINGS if (deleteAll) { - dumpValues(rawDirF, fileBase, null, null, null, null, null, -1, -1, null, true); + dumpValues(gAgencyTools, rawDirF, fileBase, null, null, null, null, null, -1, -1, null, true); } else { dumpCommonValues(rawDirF, gAgencyTools, mSpec, inputUrl); - dumpValues( + dumpValues(gAgencyTools, rawDirF, fileBase, mSpec, minMaxLatLng.first.first, minMaxLatLng.second.first, minMaxLatLng.first.second, minMaxLatLng.second.second, mSpec.getFirstTimestampInSeconds(), mSpec.getLastTimestampInSeconds(), inputUrl, @@ -1057,6 +1057,7 @@ private static String getLastModified(String gtfsDir) { private static final String GTFS_RDS_AGENCY_EXTENDED_TYPE = "gtfs_rts_agency_extended_type"; // do not change to avoid breaking compat w/ old modules public static final String GTFS_RDS_TIMEZONE = "gtfs_rts_timezone"; // do not change to avoid breaking compat w/ old modules private static final String GTFS_RDS_COLOR = "gtfs_rts_color"; // do not change to avoid breaking compat w/ old modules + private static final String GTFS_RDS_AGENCY_ID_CLEANUP_REGEX = "gtfs_rts_agency_id_cleanup_regex"; // do not change to avoid breaking compat w/ old modules private static final String GTFS_RDS_SERVICE_ID_CLEANUP_REGEX = "gtfs_rts_service_id_cleanup_regex"; // do not change to avoid breaking compat w/ old modules private static final String GTFS_RDS_ROUTE_ID_CLEANUP_REGEX = "gtfs_rts_route_id_cleanup_regex"; // do not change to avoid breaking compat w/ old modules private static final String GTFS_RDS_TRIP_ID_CLEANUP_REGEX = "gtfs_rts_trip_id_cleanup_regex"; // do not change to avoid breaking compat w/ old modules @@ -1095,7 +1096,7 @@ private static void dumpCommonValues(File dumpDirF, GAgencyTools gAgencyTools, M ow.write(Constants.NEW_LINE); } //noinspection DiscouragedApi - ow.write(getRESOURCES_STRING(GTFS_RDS_AGENCY_ID, mSpec.getFirstAgency().getId())); + ow.write(getRESOURCES_STRING(GTFS_RDS_AGENCY_ID, mSpec.getFirstAgency().getCleanAgencyId(gAgencyTools))); ow.write(Constants.NEW_LINE); ow.write(getRESOURCES_INTEGER(GTFS_RDS_AGENCY_TYPE, mSpec.getFirstAgency().getType())); ow.write(Constants.NEW_LINE); @@ -1107,6 +1108,10 @@ private static void dumpCommonValues(File dumpDirF, GAgencyTools gAgencyTools, M ow.write(Constants.NEW_LINE); ow.write(getRESOURCES_STRING(GTFS_RDS_COLOR, mSpec.getFirstAgency().getColor())); ow.write(Constants.NEW_LINE); + if (gAgencyTools.getAgencyIdCleanupRegex() != null) { + ow.write(getRESOURCES_STRING(GTFS_RDS_AGENCY_ID_CLEANUP_REGEX, escapeResString(gAgencyTools.getAgencyIdCleanupRegex()))); + ow.write(Constants.NEW_LINE); + } if (gAgencyTools.getServiceIdCleanupRegex() != null) { ow.write(getRESOURCES_STRING(GTFS_RDS_SERVICE_ID_CLEANUP_REGEX, escapeResString(gAgencyTools.getServiceIdCleanupRegex()))); ow.write(Constants.NEW_LINE); @@ -1151,7 +1156,7 @@ public static String escapeResString(@NotNull String string) { return string; } - private static void dumpValues(File dumpDirF, String fileBase, MSpec mSpec, Double minLat, Double maxLat, Double minLng, Double maxLng, + private static void dumpValues(GAgencyTools gAgencyTools, File dumpDirF, String fileBase, MSpec mSpec, Double minLat, Double maxLat, Double minLng, Double maxLng, int firstTimestampInSec, int lastTimestampInSec, @Nullable String inputUrl, boolean deleteAll) { File file; BufferedWriter ow = null; @@ -1183,7 +1188,7 @@ private static void dumpValues(File dumpDirF, String fileBase, MSpec mSpec, Doub ow.write(Constants.NEW_LINE); } //noinspection DiscouragedApi - ow.write(getRESOURCES_STRING(GTFS_RDS_AGENCY_ID, mSpec.getFirstAgency().getId())); + ow.write(getRESOURCES_STRING(GTFS_RDS_AGENCY_ID, mSpec.getFirstAgency().getCleanAgencyId(gAgencyTools))); ow.write(Constants.NEW_LINE); ow.write(getRESOURCES_INTEGER(GTFS_RDS_AGENCY_TYPE, mSpec.getFirstAgency().getType())); ow.write(Constants.NEW_LINE); diff --git a/src/main/java/org/mtransit/parser/mt/data/MAgency.kt b/src/main/java/org/mtransit/parser/mt/data/MAgency.kt index 59bcd4e2..65d67965 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MAgency.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MAgency.kt @@ -1,6 +1,6 @@ package org.mtransit.parser.mt.data -import androidx.annotation.Discouraged +import org.mtransit.parser.db.SQLUtils.escapeId import org.mtransit.parser.gtfs.GAgencyTools import org.mtransit.parser.gtfs.data.GAgency import org.mtransit.parser.gtfs.data.GIDs @@ -29,19 +29,11 @@ data class MAgency( agencyTools.agencyRouteType, ) - @Suppress("unused") - @get:Discouraged(message = "Not memory efficient") - val id: String get() = _id + fun getCleanAgencyId(agencyTools: GAgencyTools): String = _id.convertAgencyId(agencyTools) - private val _id: String - get() { - return GIDs.getString(idInt) - } + private val _id: String get() = GIDs.getString(idInt) - fun toStringPlus(): String { - return toString() + - "+(_id:$_id)" - } + fun toStringPlus() = toString() + "+(_id:$_id)" override fun compareTo(other: MAgency?): Int { return when { @@ -98,5 +90,14 @@ data class MAgency( } return null } + + internal fun convert(agencyTools: GAgencyTools, agencyId: String): String = + agencyTools.cleanAgencyId(agencyId).escapeId() } -} \ No newline at end of file +} + +fun String.convertAgencyId(agencyTools: GAgencyTools) = + MAgency.convert( + agencyTools = agencyTools, + agencyId = this, + ) \ No newline at end of file diff --git a/src/main/java/org/mtransit/parser/mt/data/MDirection.kt b/src/main/java/org/mtransit/parser/mt/data/MDirection.kt index d097b42f..62ddb697 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MDirection.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MDirection.kt @@ -190,7 +190,7 @@ data class MDirection @JvmOverloads constructor( } } - private const val SLASH = " / " + private const val SLASH = "/" @JvmStatic fun mergeHeadsignValue( diff --git a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt index 0a9e426e..2c4f1cb0 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MServiceIds.kt @@ -65,8 +65,7 @@ object MServiceIds { fun containsAllIdInts(idInts: Iterable) = idInts.all { idIntToId.containsKey(it) } - @JvmStatic - fun convert(agencyTools: GAgencyTools, serviceId: String, keep: Boolean = true, quotesString: Boolean = false): String = + internal fun convert(agencyTools: GAgencyTools, serviceId: String, keep: Boolean = true, quotesString: Boolean = false): String = if (FeatureFlags.F_EXPORT_SERVICE_ID_INTS) { getInt(agencyTools.cleanServiceId(serviceId, keep)).toString() } else { diff --git a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt index 73af2ec9..1ce0fecd 100644 --- a/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt +++ b/src/main/java/org/mtransit/parser/mt/data/MTripIds.kt @@ -43,7 +43,7 @@ object MTripIds { synchronized(incrementLock) { val iterator = idToIdInt.asMutableMap().entries.iterator() while (iterator.hasNext()) { - val (tripId,tripIdInt) = iterator.next() + val (tripId, tripIdInt) = iterator.next() if (tripId !in usedTripIds) { idIntToId.remove(tripIdInt) iterator.remove() @@ -88,8 +88,7 @@ object MTripIds { @JvmStatic fun getAllSorted() = getAll().sorted() - @JvmStatic - fun convert(tripId: String, quotesString: Boolean = false) = + internal fun convert(tripId: String, quotesString: Boolean = false) = if (FeatureFlags.F_EXPORT_TRIP_ID_INTS) { getInt(tripId).toString() } else { diff --git a/src/test/java/org/mtransit/parser/mt/MDirectionHeadSignFinderTest.kt b/src/test/java/org/mtransit/parser/mt/MDirectionHeadSignFinderTest.kt index 16cdddfa..1b118cea 100644 --- a/src/test/java/org/mtransit/parser/mt/MDirectionHeadSignFinderTest.kt +++ b/src/test/java/org/mtransit/parser/mt/MDirectionHeadSignFinderTest.kt @@ -450,7 +450,7 @@ class MDirectionHeadSignFinderTest { // Act val result = MDirectionHeadSignFinder.findDirectionHeadSign(RID, gRouteTrips, routeGTFS, directionId, agencyTools) // Assert - assertEquals("foo foo / trip head-sign", result?.headSign) + assertEquals("foo foo/trip head-sign", result?.headSign) } @Test @@ -622,7 +622,7 @@ class MDirectionHeadSignFinderTest { // Act val result = MDirectionHeadSignFinder.findDirectionHeadSign(RID, gRouteTrips, routeGTFS, directionId, agencyTools) // Assert - assertEquals("foo foo / trip head-sign", result?.headSign) + assertEquals("foo foo/trip head-sign", result?.headSign) } @Test