Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Skip license processing for top-level source directories that are unchanged #3437

Merged
jason-simmons merged 1 commit into from
Mar 1, 2017
Merged

Skip license processing for top-level source directories that are unchanged #3437

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 130 additions & 34 deletions tools/licenses/lib/main.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,16 @@

// See README in this directory for information on how this code is organised.

import 'dart:async';
import 'dart:collection';
import 'dart:convert';
import 'dart:io' as system;
import 'dart:math' as math;

import 'package:args/args.dart';
import 'package:crypto/crypto.dart' as crypto;
import 'package:path/path.dart' as path;

import 'filesystem.dart' as fs;
import 'licenses.dart';
import 'patterns.dart';
Expand Down Expand Up @@ -919,6 +924,8 @@ class RepositoryDirectory extends RepositoryEntry implements LicenseSource {
final List<RepositoryLicensedFile> _files = <RepositoryLicensedFile>[];
final List<RepositoryLicenseFile> _licenses = <RepositoryLicenseFile>[];

List<RepositoryDirectory> get subdirectories => _subdirectories;

final Map<String, RepositoryEntry> _childrenByName = <String, RepositoryEntry>{};

// the bit at the beginning excludes files like "license.py".
Expand Down Expand Up @@ -1235,6 +1242,33 @@ class RepositoryDirectory extends RepositoryEntry implements LicenseSource {
result += directory.fileCount;
return result;
}

Iterable<RepositoryLicensedFile> get _allFiles sync* {
for (RepositoryLicensedFile file in _files) {
if (file.isIncludedInBuildProducts)
yield file;
}
for (RepositoryDirectory directory in _subdirectories) {
yield* directory._allFiles;
}
}

Stream<List<int>> _signatureStream(List files) async* {
for (RepositoryLicensedFile file in files) {
yield file.io.fullName.codeUnits;
Copy link
Contributor

@chinmaygarde chinmaygarde Feb 24, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this really necessary? I am slightly worried about case sensitivity of the file system returning different results per platform. You're comparing the file contents anyway which should catch all cases.

Copy link
Member Author

@jason-simmons jason-simmons Feb 24, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is intended to catch renaming of files (which will affect the file paths listed in the output)

Copy link
Contributor

@chinmaygarde chinmaygarde Feb 24, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Oh. Good point. I forget the file names were also present in that list.

yield file.io.readBytes();
}
}

/// Compute a signature representing a hash of all the licensed files within
/// this directory tree.
Future<String> get signature async {
List allFiles = _allFiles.toList();
allFiles.sort((RepositoryLicensedFile a, RepositoryLicensedFile b) =>
a.io.fullName.compareTo(b.io.fullName));
crypto.Digest digest = await crypto.md5.bind(_signatureStream(allFiles)).single;
return digest.bytes.map((int e) => e.toRadixString(16).padLeft(2, '0')).join();
}
}

class RepositoryGenericThirdPartyDirectory extends RepositoryDirectory {
Expand Down Expand Up @@ -2237,53 +2271,115 @@ class Progress {

// MAIN

void main(List<String> arguments) {
if (arguments.length != 1) {
print('Usage: dart lib/main.dart path/to/engine/root/src');
Future<Null> main(List<String> arguments) async {
final ArgParser parser = new ArgParser()
..addOption('src', help: 'The root of the engine source')
..addOption('out', help: 'The directory where output is written')
..addOption('golden', help: 'The directory containing golden results')
..addFlag('release', help: 'Print output in the format used for product releases');
Copy link
Contributor

@chinmaygarde chinmaygarde Feb 24, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for these flags. It makes the usage so much clearer.


ArgResults argResults = parser.parse(arguments);
bool releaseMode = argResults['release'];
if (argResults['src'] == null) {
print('Flutter license script: Must provide --src directory');
print(parser.usage);
system.exit(1);
}
if (!releaseMode) {
if (argResults['out'] == null || argResults['golden'] == null) {
print('Flutter license script: Must provide --out and --golden directories in non-release mode');
print(parser.usage);
system.exit(1);
}
if (!system.FileSystemEntity.isDirectorySync(argResults['golden'])) {
print('Flutter license script: Golden directory does not exist');
print(parser.usage);
system.exit(1);
}
system.Directory out = new system.Directory(argResults['out']);
if (!out.existsSync())
out.createSync(recursive: true);
}

try {
system.stderr.writeln('Finding files...');
final RepositoryDirectory root = new RepositoryRoot(new fs.FileSystemDirectory.fromPath(arguments.single));
system.stderr.writeln('Collecting licenses...');
Progress progress = new Progress(root.fileCount);
List<License> licenses = new Set<License>.from(root.getLicenses(progress).toList()).toList();
progress.label = 'Dumping results...';
bool done = false;
List<License> usedLicenses = licenses.where((License license) => license.isUsed).toList();
assert(() {
print('UNUSED LICENSES:\n');
List<String> unusedLicenses = licenses
.where((License license) => !license.isUsed)
.map((License license) => license.toString())
.toList();
unusedLicenses.sort();
print(unusedLicenses.join('\n\n'));
print('~' * 80);
print('USED LICENSES:\n');
List<String> output = usedLicenses.map((License license) => license.toString()).toList();
output.sort();
print(output.join('\n\n'));
done = true;
return true;
});
if (!done) {
final RepositoryDirectory root = new RepositoryRoot(new fs.FileSystemDirectory.fromPath(argResults['src']));

if (releaseMode) {
system.stderr.writeln('Collecting licenses...');
Progress progress = new Progress(root.fileCount);
List<License> licenses = new Set<License>.from(root.getLicenses(progress).toList()).toList();
if (progress.hadErrors)
throw 'Had failures while collecting licenses.';
List<String> output = usedLicenses
progress.label = 'Dumping results...';
List<String> output = licenses
.where((License license) => license.isUsed)
.map((License license) => license.toStringFormal())
.where((String text) => text != null)
.toList();
output.sort();
print(output.join('\n${"-" * 80}\n'));
} else {
RegExp signaturePattern = new RegExp(r'Signature: (\w+)');

for (RepositoryDirectory component in root.subdirectories) {
system.stderr.writeln('Collecting licenses for ${component.io.name}');

String signature;
if (component.io.name == 'flutter') {
// Always run the full license check on the flutter tree. This tree is
// relatively small but changes frequently in ways that do not affect
// the license output, and we don't want to require updates to the golden
// signature for those changes.
signature = null;
} else {
signature = await component.signature;
}

// Check whether the golden file matches the signature of the current contents
// of this directory.
system.File goldenFile = new system.File(
path.join(argResults['golden'], 'licenses_${component.io.name}'));
String goldenSignature = await goldenFile.openRead()
.transform(UTF8.decoder).transform(new LineSplitter()).first;
Match goldenMatch = signaturePattern.matchAsPrefix(goldenSignature);
if (goldenMatch != null && goldenMatch.group(1) == signature) {
system.stderr.writeln(' Skipping this component - no change in signature');
continue;
}

Progress progress = new Progress(component.fileCount);

system.File outFile = new system.File(
path.join(argResults['out'], 'licenses_${component.io.name}'));
system.IOSink sink = outFile.openWrite();
if (signature != null)
sink.writeln('Signature: $signature\n');

List<License> licenses = new Set<License>.from(
component.getLicenses(progress).toList()).toList();

sink.writeln('UNUSED LICENSES:\n');
List<String> unusedLicenses = licenses
.where((License license) => !license.isUsed)
.map((License license) => license.toString())
.toList();
unusedLicenses.sort();
sink.writeln(unusedLicenses.join('\n\n'));
sink.writeln('~' * 80);

sink.writeln('USED LICENSES:\n');
List<License> usedLicenses = licenses.where((License license) => license.isUsed).toList();
List<String> output = usedLicenses.map((License license) => license.toString()).toList();
output.sort();
sink.writeln(output.join('\n\n'));
sink.writeln('Total license count: ${licenses.length}');

await sink.close();
progress.label = 'Done.';
system.stderr.writeln('');
}
}
assert(() {
print('Total license count: ${licenses.length}');
progress.label = 'Done.';
print('$progress');
return true;
});
} catch (e, stack) {
system.stderr.writeln('failure: $e\n$stack');
system.stderr.writeln('aborted.');
Expand Down
2 changes: 1 addition & 1 deletion tools/licenses/pubspec.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ dependencies:
path: ^1.3.0
archive: ^1.0.24
args: 0.13.7

crypto: ^2.0.1
38 changes: 21 additions & 17 deletions travis/licenses.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,23 @@
shopt -s nullglob

echo "Verifying license script is still happy..."
(cd flutter/tools/licenses; pub get; dart --checked lib/main.dart ../../.. > ../../../out/license-script-output)
(cd flutter/tools/licenses; pub get; dart --checked lib/main.dart --src ../../.. --out ../../../out/license_script_output --golden ../../travis/licenses_golden)

for f in out/license_script_output/licenses_*; do
if ! cmp -s flutter/travis/licenses_golden/$(basename $f) $f
then
echo "License script got different results than expected for $f."
echo "Please rerun the licenses script locally to verify that it is"
echo "correctly catching any new licenses for anything you may have"
echo "changed, and then update this file:"
echo " flutter/sky/packages/sky_engine/LICENSE"
echo "For more information, see the script in:"
echo " https://github.com/flutter/engine/tree/master/tools/licenses"
echo ""
diff -U 6 flutter/travis/licenses_golden/$(basename $f) $f
exit 1
fi
done

if cmp -s flutter/travis/licenses.golden out/license-script-output
then
echo "Licenses are as expected."
exit 0
else
echo "License script got different results than expected."
echo "Please rerun the licenses script locally to verify that it is"
echo "correctly catching any new licenses for anything you may have"
echo "changed, and then update this file:"
echo " flutter/sky/packages/sky_engine/LICENSE"
echo "For more information, see the script in:"
echo " https://github.com/flutter/engine/tree/master/tools/licenses"
echo ""
diff -U 6 flutter/travis/licenses.golden out/license-script-output
exit 1
fi
echo "Licenses are as expected."
exit 0
Loading