From 88da391db36a4f5acb2097c3f5f8e1bbffee1aff Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Sun, 12 Jun 2022 02:01:59 +0000 Subject: [PATCH 01/11] feat: Increase initial map capacity --- third_party/docfx-doclet-143274/pom.xml | 2 +- .../src/main/java/com/microsoft/lookup/BaseLookup.java | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/third_party/docfx-doclet-143274/pom.xml b/third_party/docfx-doclet-143274/pom.xml index 9e53477f..57c9d96d 100644 --- a/third_party/docfx-doclet-143274/pom.xml +++ b/third_party/docfx-doclet-143274/pom.xml @@ -6,7 +6,7 @@ com.microsoft docfx-doclet - 1.0-SNAPSHOT + 1.7.0 UTF-8 diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java index 2d0ede47..8302dedc 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java @@ -44,7 +44,7 @@ public abstract class BaseLookup { put(ElementKind.FIELD, "Field"); }}; - protected Map map = new HashMap<>(); + protected Map map = new HashMap<>(10000); protected final DocletEnvironment environment; protected BaseLookup(DocletEnvironment environment) { @@ -52,12 +52,8 @@ protected BaseLookup(DocletEnvironment environment) { } protected ExtendedMetadataFileItem resolve(T key) { - ExtendedMetadataFileItem value = map.get(key); - if (value == null) { - value = buildMetadataFileItem(key); - map.put(key, value); - } - return value; + map.computeIfAbsent(key, this::buildMetadataFileItem); + return map.get(key); } protected abstract ExtendedMetadataFileItem buildMetadataFileItem(T key); From d1a4846a76c5d044bc1113012b23cbea0542b1db Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Sun, 12 Jun 2022 03:59:08 +0000 Subject: [PATCH 02/11] feat: Add thread safe element access --- .../com/microsoft/build/ClassBuilder.java | 8 +-- .../com/microsoft/build/YmlFilesBuilder.java | 62 ++++++++++++++----- .../com/microsoft/doclet/DocFxDoclet.java | 2 +- .../java/com/microsoft/lookup/BaseLookup.java | 3 +- .../microsoft/lookup/ClassItemsLookup.java | 8 +-- .../com/microsoft/lookup/ClassLookup.java | 8 ++- .../java/com/microsoft/model/TocFile.java | 2 +- .../java/com/microsoft/util/ElementUtil.java | 11 +++- .../main/java/com/microsoft/util/Utils.java | 2 +- .../microsoft/build/YmlFilesBuilderTest.java | 2 +- 10 files changed, 77 insertions(+), 31 deletions(-) diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java index c70652f7..179ed543 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java @@ -102,7 +102,7 @@ private void addClassInfo(TypeElement classElement, MetadataFile classMetadataFi } void addConstructorsInfo(TypeElement classElement, MetadataFile classMetadataFile) { - for (ExecutableElement constructorElement : ElementFilter.constructorsIn(classElement.getEnclosedElements())) { + for (ExecutableElement constructorElement : ElementFilter.constructorsIn(ElementUtil.getEnclosedElements(classElement))) { MetadataFileItem constructorItem = buildMetadataFileItem(constructorElement); constructorItem.setOverload(classItemsLookup.extractOverload(constructorElement)); constructorItem.setContent(classItemsLookup.extractConstructorContent(constructorElement)); @@ -115,7 +115,7 @@ void addConstructorsInfo(TypeElement classElement, MetadataFile classMetadataFil } private void addMethodsInfo(TypeElement classElement, MetadataFile classMetadataFile) { - ElementFilter.methodsIn(classElement.getEnclosedElements()).stream() + ElementFilter.methodsIn(ElementUtil.getEnclosedElements(classElement)).stream() .filter(methodElement -> !Utils.isPrivateOrPackagePrivate(methodElement)) .forEach(methodElement -> { MetadataFileItem methodItem = buildMetadataFileItem(methodElement); @@ -135,7 +135,7 @@ private void addMethodsInfo(TypeElement classElement, MetadataFile classMetadata } private void addFieldsInfo(TypeElement classElement, MetadataFile classMetadataFile) { - ElementFilter.fieldsIn(classElement.getEnclosedElements()).stream() + ElementFilter.fieldsIn(ElementUtil.getEnclosedElements(classElement)).stream() .filter(fieldElement -> !Utils.isPrivateOrPackagePrivate(fieldElement)) .forEach(fieldElement -> { MetadataFileItem fieldItem = buildMetadataFileItem(fieldElement); @@ -177,7 +177,7 @@ private void collect(TypeElement classElement, List children, Function, List> selectFunc, Function mapFunc) { - List elements = selectFunc.apply(classElement.getEnclosedElements()); + List elements = selectFunc.apply(ElementUtil.getEnclosedElements(classElement)); children.addAll(filterPrivateElements(elements).stream() .map(mapFunc).collect(Collectors.toList())); } diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java index b558d1a6..94785d92 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java @@ -10,12 +10,20 @@ import com.microsoft.model.TocTypeMap; import com.microsoft.util.ElementUtil; import com.microsoft.util.FileUtil; +import java.util.Collections; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import javax.tools.Diagnostic.Kind; import jdk.javadoc.doclet.DocletEnvironment; import javax.lang.model.element.PackageElement; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import jdk.javadoc.doclet.Reporter; import static com.microsoft.build.BuilderUtil.populateUidValues; @@ -30,9 +38,10 @@ public class YmlFilesBuilder { private PackageBuilder packageBuilder; private ClassBuilder classBuilder; private ReferenceBuilder referenceBuilder; + private Reporter reporter; public YmlFilesBuilder(DocletEnvironment environment, String outputPath, - String[] excludePackages, String[] excludeClasses, String projectName, boolean disableChangelog) { + String[] excludePackages, String[] excludeClasses, String projectName, boolean disableChangelog, Reporter reporter) { this.environment = environment; this.outputPath = outputPath; this.elementUtil = new ElementUtil(excludePackages, excludeClasses); @@ -43,10 +52,12 @@ public YmlFilesBuilder(DocletEnvironment environment, String outputPath, ClassLookup classLookup = new ClassLookup(environment); this.referenceBuilder = new ReferenceBuilder(environment, classLookup, elementUtil); this.packageBuilder = new PackageBuilder(packageLookup, outputPath, referenceBuilder); + this.reporter = reporter; this.classBuilder = new ClassBuilder(elementUtil, classLookup, new ClassItemsLookup(environment), outputPath, referenceBuilder); } public boolean build() { + ExecutorService executorService = Executors.newFixedThreadPool(10); // table of contents TocFile tocFile = new TocFile(outputPath, projectName, disableChangelog); // overview page @@ -56,23 +67,46 @@ public boolean build() { // packages List packageItems = new ArrayList<>(); // class/enum/interface/etc. pages - List classMetadataFiles = new ArrayList<>(); + List classMetadataFiles = Collections.synchronizedList(new ArrayList<>()); + List> futureList = new ArrayList<>(); for (PackageElement packageElement : elementUtil.extractPackageElements(environment.getIncludedElements())) { - String packageUid = packageLookup.extractUid(packageElement); - String packageStatus = packageLookup.extractStatus(packageElement); - TocItem packageTocItem = new TocItem(packageUid, packageUid, packageStatus); - // build package summary - packageMetadataFiles.add(packageBuilder.buildPackageMetadataFile(packageElement)); - // add package summary to toc - packageTocItem.getItems().add(new TocItem(packageUid, "Package summary")); - tocFile.addTocItem(packageTocItem); + Future future = executorService.submit(() -> { + reporter.print(Kind.NOTE, "Running for " + packageElement + " on " + Thread.currentThread().getName()); + String packageUid = packageLookup.extractUid(packageElement); + String packageStatus = packageLookup.extractStatus(packageElement); + TocItem packageTocItem = new TocItem(packageUid, packageUid, packageStatus); + // build package summary + packageMetadataFiles.add(packageBuilder.buildPackageMetadataFile(packageElement)); + // add package summary to toc + packageTocItem.getItems().add(new TocItem(packageUid, "Package summary")); + tocFile.addTocItem(packageTocItem); - // build classes/interfaces/enums/exceptions/annotations - TocTypeMap typeMap = new TocTypeMap(); - classBuilder.buildFilesForInnerClasses(packageElement, typeMap, classMetadataFiles); - packageTocItem.getItems().addAll(joinTocTypeItems(typeMap)); + // build classes/interfaces/enums/exceptions/annotations + TocTypeMap typeMap = new TocTypeMap(); + classBuilder.buildFilesForInnerClasses(packageElement, typeMap, classMetadataFiles); + packageTocItem.getItems().addAll(joinTocTypeItems(typeMap)); + reporter.print(Kind.NOTE, "Finished running for " + packageElement + " on " + Thread.currentThread().getName()); + }); + futureList.add(future); + } + + for (Future future : futureList) { + try { + future.get(); + } catch (InterruptedException e) { + e.printStackTrace(); + } catch (ExecutionException e) { + e.printStackTrace(); + } + } + + executorService.shutdown(); + try { + executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.MICROSECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); } for (MetadataFile packageFile : packageMetadataFiles) { diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocFxDoclet.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocFxDoclet.java index 20050e74..e839b9b4 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocFxDoclet.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocFxDoclet.java @@ -28,7 +28,7 @@ public boolean run(DocletEnvironment environment) { reporter.print(Kind.NOTE, "Project name: " + projectName); reporter.print(Kind.NOTE, "Disable changelog: " + disableChangelog); - return (new YmlFilesBuilder(environment, outputPath, excludePackages, excludeClasses, projectName, disableChangelog)).build(); + return (new YmlFilesBuilder(environment, outputPath, excludePackages, excludeClasses, projectName, disableChangelog, reporter)).build(); } @Override diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java index 8302dedc..e0bf6480 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java @@ -13,6 +13,7 @@ import com.sun.source.doctree.LinkTree; import com.sun.source.doctree.LiteralTree; import com.sun.source.doctree.SeeTree; +import java.util.concurrent.ConcurrentHashMap; import jdk.javadoc.doclet.DocletEnvironment; import org.apache.commons.lang3.RegExUtils; import org.apache.commons.lang3.StringUtils; @@ -44,7 +45,7 @@ public abstract class BaseLookup { put(ElementKind.FIELD, "Field"); }}; - protected Map map = new HashMap<>(10000); + protected Map map = new ConcurrentHashMap<>(10000); protected final DocletEnvironment environment; protected BaseLookup(DocletEnvironment environment) { diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassItemsLookup.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassItemsLookup.java index d3e8d2a3..1ea610d8 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassItemsLookup.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassItemsLookup.java @@ -35,7 +35,7 @@ public ClassItemsLookup(DocletEnvironment environment) { } @Override - protected ExtendedMetadataFileItem buildMetadataFileItem(Element element) { + protected synchronized ExtendedMetadataFileItem buildMetadataFileItem(Element element) { String packageName = determinePackageName(element); TypeElement classElement = (TypeElement) element.getEnclosingElement(); String classQName = String.valueOf(classElement.getQualifiedName()); @@ -59,14 +59,14 @@ protected ExtendedMetadataFileItem buildMetadataFileItem(Element element) { ExecutableElement exeElement = (ExecutableElement) element; List parameters = extractParameters(exeElement); String paramsString = parameters.stream() - .map(parameter -> String.format("%s %s", makeTypeShort(parameter.getType()), parameter.getId())) - .collect(Collectors.joining(", ")); + .map(parameter -> String.format("%s %s", makeTypeShort(parameter.getType()), parameter.getId())) + .collect(Collectors.joining(", ")); String nameWithoutBrackets = elementQName.replaceAll("\\(.*\\)", ""); String methodName = String.format("%s(%s)", nameWithoutBrackets, paramsString); result.setName(methodName); result.setMethodContent(String.format("%s %s %s", modifiers, - makeTypeShort(String.valueOf(exeElement.getReturnType())), result.getName())); + makeTypeShort(String.valueOf(exeElement.getReturnType())), result.getName())); result.setConstructorContent(String.format("%s %s", modifiers, result.getName())); result.setParameters(parameters); result.setExceptions(extractExceptions(exeElement)); diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java index 91f097cc..70f9dcfa 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java @@ -3,7 +3,9 @@ import com.microsoft.lookup.model.ExtendedMetadataFileItem; import com.microsoft.model.MetadataFileItem; import com.microsoft.model.TypeParameter; +import com.microsoft.util.ElementUtil; import com.microsoft.util.Utils; +import java.util.Collections; import jdk.javadoc.doclet.DocletEnvironment; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -30,8 +32,8 @@ public ClassLookup(DocletEnvironment environment) { } @Override - protected ExtendedMetadataFileItem buildMetadataFileItem(TypeElement classElement) { - List inheritedMethods = new ArrayList<>(); + protected synchronized ExtendedMetadataFileItem buildMetadataFileItem(TypeElement classElement) { + List inheritedMethods = Collections.synchronizedList(new ArrayList<>()); String packageName = determinePackageName(classElement); String classQName = String.valueOf(classElement.getQualifiedName()); @@ -146,7 +148,7 @@ List determineTypeParameters(TypeElement element) { } void appendInheritedMethods(TypeElement element, List inheritedMethods) { - List members = element.getEnclosedElements(); + List members = ElementUtil.getEnclosedElements(element); Integer level = Optional.ofNullable(getMaxNestedLevel(inheritedMethods)) .orElse(0); diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/model/TocFile.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/model/TocFile.java index e32af29b..fe67c864 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/model/TocFile.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/model/TocFile.java @@ -22,7 +22,7 @@ public TocFile(String outputPath, String projectName, boolean disableChangelog) this.disableChangelog = disableChangelog; } - public void addTocItem(TocItem packageTocItem) { + public synchronized void addTocItem(TocItem packageTocItem) { add(packageTocItem); } diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/ElementUtil.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/ElementUtil.java index 54b8ac64..7706a634 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/ElementUtil.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/ElementUtil.java @@ -1,5 +1,7 @@ package com.microsoft.util; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.apache.commons.lang3.StringUtils; import javax.lang.model.element.Element; @@ -18,6 +20,8 @@ public class ElementUtil { private final Set excludePackages = new HashSet<>(); private final Set excludeClasses = new HashSet<>(); + private static final Map> elementMap = new ConcurrentHashMap<>(10000); + public ElementUtil(String[] excludePackages, String[] excludeClasses) { this.excludePackages.addAll(Stream.of(excludePackages) .map(o -> Pattern.compile(o)).collect(Collectors.toSet())); @@ -29,7 +33,7 @@ public List extractSortedElements(Element element) { // Need to apply sorting, because order of result items for Element.getEnclosedElements() depend on JDK implementation // By default, exclude private and package-private items // todo allow pass parameter for filter items by access modifiers - return ElementFilter.typesIn(element.getEnclosedElements()).stream() + return ElementFilter.typesIn(getEnclosedElements(element)).stream() .filter(o -> !Utils.isPrivateOrPackagePrivate(o)) .filter(o -> !matchAnyPattern(excludeClasses, String.valueOf(o.getQualifiedName()))) .sorted((o1, o2) -> @@ -37,6 +41,11 @@ public List extractSortedElements(Element element) { ).collect(Collectors.toList()); } + public static synchronized List getEnclosedElements(Element element) { + elementMap.computeIfAbsent(element, Element::getEnclosedElements); + return elementMap.get(element); + } + public List extractPackageElements(Set elements) { return ElementFilter.packagesIn(elements).stream() .filter(o -> !matchAnyPattern(excludePackages, String.valueOf(o))) diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/Utils.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/Utils.java index ba155915..c5ead09a 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/Utils.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/Utils.java @@ -219,7 +219,7 @@ public List removeBlockTag(List dctree, Do * @return a list of visible enclosed members in this type */ public List getMembers(TypeElement te, ElementKind kind) { - return te.getEnclosedElements().stream() + return ElementUtil.getEnclosedElements(te).stream() .filter(e -> e.getKind() == kind && !isPrivateOrPackagePrivate(e)) .collect(Collectors.toList()); } diff --git a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/YmlFilesBuilderTest.java b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/YmlFilesBuilderTest.java index 027ba7d7..1a2db880 100644 --- a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/YmlFilesBuilderTest.java +++ b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/YmlFilesBuilderTest.java @@ -23,7 +23,7 @@ public class YmlFilesBuilderTest { @Before public void setup() { environment = Mockito.mock(DocletEnvironment.class); - ymlFilesBuilder = new YmlFilesBuilder(environment, "./target", new String[]{}, new String[]{}, "google-cloud-product", false); + ymlFilesBuilder = new YmlFilesBuilder(environment, "./target", new String[]{}, new String[]{}, "google-cloud-product", false, null); } @Test From 7cdc0a0be9774908b3876a04a0d91211d4bb4e70 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 13 Jun 2022 13:11:49 +0000 Subject: [PATCH 03/11] feat: Use common map for caching results --- .../com/microsoft/build/ClassBuilder.java | 8 +++--- .../com/microsoft/build/YmlFilesBuilder.java | 4 +-- .../java/com/microsoft/lookup/BaseLookup.java | 2 +- .../microsoft/lookup/ClassItemsLookup.java | 5 ++-- .../com/microsoft/lookup/ClassLookup.java | 9 ++++-- .../java/com/microsoft/util/ElementUtil.java | 28 +++++++++++++------ .../main/java/com/microsoft/util/Utils.java | 14 ++++++---- .../com/microsoft/build/ClassBuilderTest.java | 4 +-- .../microsoft/build/ReferenceBuilderTest.java | 2 +- .../lookup/ClassItemsLookupTest.java | 2 +- .../com/microsoft/lookup/ClassLookupTest.java | 2 +- 11 files changed, 49 insertions(+), 31 deletions(-) diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java index 179ed543..efcd26ec 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java @@ -102,7 +102,7 @@ private void addClassInfo(TypeElement classElement, MetadataFile classMetadataFi } void addConstructorsInfo(TypeElement classElement, MetadataFile classMetadataFile) { - for (ExecutableElement constructorElement : ElementFilter.constructorsIn(ElementUtil.getEnclosedElements(classElement))) { + for (ExecutableElement constructorElement : ElementFilter.constructorsIn(elementUtil.getEnclosedElements(classElement))) { MetadataFileItem constructorItem = buildMetadataFileItem(constructorElement); constructorItem.setOverload(classItemsLookup.extractOverload(constructorElement)); constructorItem.setContent(classItemsLookup.extractConstructorContent(constructorElement)); @@ -115,7 +115,7 @@ void addConstructorsInfo(TypeElement classElement, MetadataFile classMetadataFil } private void addMethodsInfo(TypeElement classElement, MetadataFile classMetadataFile) { - ElementFilter.methodsIn(ElementUtil.getEnclosedElements(classElement)).stream() + ElementFilter.methodsIn(elementUtil.getEnclosedElements(classElement)).stream() .filter(methodElement -> !Utils.isPrivateOrPackagePrivate(methodElement)) .forEach(methodElement -> { MetadataFileItem methodItem = buildMetadataFileItem(methodElement); @@ -135,7 +135,7 @@ private void addMethodsInfo(TypeElement classElement, MetadataFile classMetadata } private void addFieldsInfo(TypeElement classElement, MetadataFile classMetadataFile) { - ElementFilter.fieldsIn(ElementUtil.getEnclosedElements(classElement)).stream() + ElementFilter.fieldsIn(elementUtil.getEnclosedElements(classElement)).stream() .filter(fieldElement -> !Utils.isPrivateOrPackagePrivate(fieldElement)) .forEach(fieldElement -> { MetadataFileItem fieldItem = buildMetadataFileItem(fieldElement); @@ -177,7 +177,7 @@ private void collect(TypeElement classElement, List children, Function, List> selectFunc, Function mapFunc) { - List elements = selectFunc.apply(ElementUtil.getEnclosedElements(classElement)); + List elements = selectFunc.apply(elementUtil.getEnclosedElements(classElement)); children.addAll(filterPrivateElements(elements).stream() .map(mapFunc).collect(Collectors.toList())); } diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java index 94785d92..f4604d92 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java @@ -49,11 +49,11 @@ public YmlFilesBuilder(DocletEnvironment environment, String outputPath, this.projectName = projectName; this.disableChangelog = disableChangelog; this.projectBuilder = new ProjectBuilder(projectName); - ClassLookup classLookup = new ClassLookup(environment); + ClassLookup classLookup = new ClassLookup(environment, elementUtil); this.referenceBuilder = new ReferenceBuilder(environment, classLookup, elementUtil); this.packageBuilder = new PackageBuilder(packageLookup, outputPath, referenceBuilder); this.reporter = reporter; - this.classBuilder = new ClassBuilder(elementUtil, classLookup, new ClassItemsLookup(environment), outputPath, referenceBuilder); + this.classBuilder = new ClassBuilder(elementUtil, classLookup, new ClassItemsLookup(environment, elementUtil), outputPath, referenceBuilder); } public boolean build() { diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java index e0bf6480..fb7dc0d7 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java @@ -45,7 +45,7 @@ public abstract class BaseLookup { put(ElementKind.FIELD, "Field"); }}; - protected Map map = new ConcurrentHashMap<>(10000); + protected Map map = new ConcurrentHashMap<>(500000); protected final DocletEnvironment environment; protected BaseLookup(DocletEnvironment environment) { diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassItemsLookup.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassItemsLookup.java index 1ea610d8..c49a8115 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassItemsLookup.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassItemsLookup.java @@ -5,6 +5,7 @@ import com.microsoft.model.MethodParameter; import com.microsoft.model.Return; import com.microsoft.util.CommentHelper; +import com.microsoft.util.ElementUtil; import com.microsoft.util.Utils; import com.sun.source.doctree.DocCommentTree; import com.sun.source.doctree.DocTree; @@ -29,9 +30,9 @@ public class ClassItemsLookup extends BaseLookup { private Utils utils; - public ClassItemsLookup(DocletEnvironment environment) { + public ClassItemsLookup(DocletEnvironment environment, ElementUtil elementUtil) { super(environment); - utils = new Utils(environment); + utils = new Utils(environment, elementUtil); } @Override diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java index 70f9dcfa..9c8fa867 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java @@ -27,13 +27,16 @@ public class ClassLookup extends BaseLookup { private static final String JAVA_LANG_OBJECT = "java.lang.Object"; - public ClassLookup(DocletEnvironment environment) { + private ElementUtil elementUtil; + + public ClassLookup(DocletEnvironment environment, ElementUtil elementUtil) { super(environment); + this.elementUtil = elementUtil; } @Override protected synchronized ExtendedMetadataFileItem buildMetadataFileItem(TypeElement classElement) { - List inheritedMethods = Collections.synchronizedList(new ArrayList<>()); + List inheritedMethods = new ArrayList<>(); String packageName = determinePackageName(classElement); String classQName = String.valueOf(classElement.getQualifiedName()); @@ -148,7 +151,7 @@ List determineTypeParameters(TypeElement element) { } void appendInheritedMethods(TypeElement element, List inheritedMethods) { - List members = ElementUtil.getEnclosedElements(element); + List members = elementUtil.getEnclosedElements(element); Integer level = Optional.ofNullable(getMaxNestedLevel(inheritedMethods)) .orElse(0); diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/ElementUtil.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/ElementUtil.java index 7706a634..2ab24c52 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/ElementUtil.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/ElementUtil.java @@ -20,32 +20,44 @@ public class ElementUtil { private final Set excludePackages = new HashSet<>(); private final Set excludeClasses = new HashSet<>(); - private static final Map> elementMap = new ConcurrentHashMap<>(10000); + private final Map> elementMap; + private final Map> elementSortedMap; public ElementUtil(String[] excludePackages, String[] excludeClasses) { this.excludePackages.addAll(Stream.of(excludePackages) .map(o -> Pattern.compile(o)).collect(Collectors.toSet())); this.excludeClasses.addAll(Stream.of(excludeClasses) .map(o -> Pattern.compile(o)).collect(Collectors.toSet())); + this.elementMap = new ConcurrentHashMap<>(500000); + this.elementSortedMap = new ConcurrentHashMap<>(500000); } public List extractSortedElements(Element element) { + elementSortedMap.computeIfAbsent(element, this::extractSortedElementsInternal); + return elementSortedMap.get(element); + } + + private List extractSortedElementsInternal(Element element) { // Need to apply sorting, because order of result items for Element.getEnclosedElements() depend on JDK implementation // By default, exclude private and package-private items // todo allow pass parameter for filter items by access modifiers return ElementFilter.typesIn(getEnclosedElements(element)).stream() - .filter(o -> !Utils.isPrivateOrPackagePrivate(o)) - .filter(o -> !matchAnyPattern(excludeClasses, String.valueOf(o.getQualifiedName()))) - .sorted((o1, o2) -> - StringUtils.compare(String.valueOf(o1.getSimpleName()), String.valueOf(o2.getSimpleName())) - ).collect(Collectors.toList()); + .filter(o -> !Utils.isPrivateOrPackagePrivate(o)) + .filter(o -> !matchAnyPattern(excludeClasses, String.valueOf(o.getQualifiedName()))) + .sorted((o1, o2) -> + StringUtils.compare(String.valueOf(o1.getSimpleName()), String.valueOf(o2.getSimpleName())) + ).collect(Collectors.toList()); } - public static synchronized List getEnclosedElements(Element element) { - elementMap.computeIfAbsent(element, Element::getEnclosedElements); + public List getEnclosedElements(Element element) { + elementMap.computeIfAbsent(element, this::getEnclosedElementsInternal); return elementMap.get(element); } + private synchronized List getEnclosedElementsInternal(Element element) { + return element.getEnclosedElements(); + } + public List extractPackageElements(Set elements) { return ElementFilter.packagesIn(elements).stream() .filter(o -> !matchAnyPattern(excludePackages, String.valueOf(o))) diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/Utils.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/Utils.java index c5ead09a..019a3854 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/Utils.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/Utils.java @@ -21,13 +21,15 @@ public class Utils { public final DocletEnvironment docletEnvironment; - public final Elements elementUtils; + public final Elements elements; public final Types typeUtils; + private final ElementUtil elementUtil; - public Utils(DocletEnvironment docEnv) { + public Utils(DocletEnvironment docEnv, ElementUtil elementUtil) { docletEnvironment = docEnv; - elementUtils = docEnv.getElementUtils(); + elements = docEnv.getElementUtils(); typeUtils = docEnv.getTypeUtils(); + this.elementUtil = elementUtil; } public static boolean isPackagePrivate(Element e) { @@ -90,7 +92,7 @@ public Element getMemberBySignature(TypeElement te, ElementKind kind, String sig } public TypeElement getObjectType() { - return elementUtils.getTypeElement("java.lang.Object"); + return elements.getTypeElement("java.lang.Object"); } /** @@ -147,7 +149,7 @@ public ExecutableElement overriddenMethod(ExecutableElement method) { for (Element e : getMembers(te, ElementKind.METHOD)) { ExecutableElement ee = (ExecutableElement) e; - if (elementUtils.overrides(method, ee, origin) + if (elements.overrides(method, ee, origin) ) { return ee; } @@ -219,7 +221,7 @@ public List removeBlockTag(List dctree, Do * @return a list of visible enclosed members in this type */ public List getMembers(TypeElement te, ElementKind kind) { - return ElementUtil.getEnclosedElements(te).stream() + return elementUtil.getEnclosedElements(te).stream() .filter(e -> e.getKind() == kind && !isPrivateOrPackagePrivate(e)) .collect(Collectors.toList()); } diff --git a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/ClassBuilderTest.java b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/ClassBuilderTest.java index 1a9d1ef0..7d2ec142 100644 --- a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/ClassBuilderTest.java +++ b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/ClassBuilderTest.java @@ -50,11 +50,11 @@ public void setup() { environment = Mockito.mock(DocletEnvironment.class); docTrees = Mockito.mock(DocTrees.class); ElementUtil elementUtil = new ElementUtil(new String[0], new String[0]); - ClassLookup classLookup = new ClassLookup(environment); + ClassLookup classLookup = new ClassLookup(environment, elementUtil); classBuilder = new ClassBuilder( elementUtil, classLookup, - new ClassItemsLookup(environment), + new ClassItemsLookup(environment, null), "./target", new ReferenceBuilder(environment, classLookup, elementUtil)); } diff --git a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/ReferenceBuilderTest.java b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/ReferenceBuilderTest.java index 4afc21d2..23390b0a 100644 --- a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/ReferenceBuilderTest.java +++ b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/ReferenceBuilderTest.java @@ -40,7 +40,7 @@ public class ReferenceBuilderTest { public void setup() { DocletEnvironment environment = Mockito.mock(DocletEnvironment.class); ElementUtil elementUtil = new ElementUtil(new String[0], new String[0]); - ClassLookup classLookup = new ClassLookup(environment); + ClassLookup classLookup = new ClassLookup(environment, elementUtil); referenceBuilder = new ReferenceBuilder(environment, classLookup, elementUtil); } diff --git a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/lookup/ClassItemsLookupTest.java b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/lookup/ClassItemsLookupTest.java index 2f990ade..cde649de 100644 --- a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/lookup/ClassItemsLookupTest.java +++ b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/lookup/ClassItemsLookupTest.java @@ -73,7 +73,7 @@ public void setup() { deprecatedTree = Mockito.mock(DeprecatedTree.class); textTree = Mockito.mock(TextTree.class); identifierTree = Mockito.mock(IdentifierTree.class); - classItemsLookup = new ClassItemsLookup(environment); + classItemsLookup = new ClassItemsLookup(environment, null); } diff --git a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/lookup/ClassLookupTest.java b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/lookup/ClassLookupTest.java index 9ec0557b..dcf1b65a 100644 --- a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/lookup/ClassLookupTest.java +++ b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/lookup/ClassLookupTest.java @@ -46,7 +46,7 @@ public class ClassLookupTest { public void setup() { elements = rule.getElements(); environment = Mockito.mock(DocletEnvironment.class); - classLookup = new ClassLookup(environment); + classLookup = new ClassLookup(environment, null); docTrees = Mockito.mock(DocTrees.class); docTree = Mockito.mock(DocTree.class); docCommentTree = Mockito.mock(DocCommentTree.class); From 2baa68bdd5653fcb6695fe309be43d91ae2a201f Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 13 Jun 2022 16:42:15 +0000 Subject: [PATCH 04/11] feat: Build overview.yml file in order --- .../com/microsoft/build/YmlFilesBuilder.java | 19 +++++++++---------- .../java/com/microsoft/model/TocFile.java | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java index f4604d92..aed03646 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java @@ -72,17 +72,16 @@ public boolean build() { List> futureList = new ArrayList<>(); for (PackageElement packageElement : elementUtil.extractPackageElements(environment.getIncludedElements())) { - Future future = executorService.submit(() -> { - reporter.print(Kind.NOTE, "Running for " + packageElement + " on " + Thread.currentThread().getName()); - String packageUid = packageLookup.extractUid(packageElement); - String packageStatus = packageLookup.extractStatus(packageElement); - TocItem packageTocItem = new TocItem(packageUid, packageUid, packageStatus); - // build package summary - packageMetadataFiles.add(packageBuilder.buildPackageMetadataFile(packageElement)); - // add package summary to toc - packageTocItem.getItems().add(new TocItem(packageUid, "Package summary")); - tocFile.addTocItem(packageTocItem); + String packageUid = packageLookup.extractUid(packageElement); + String packageStatus = packageLookup.extractStatus(packageElement); + TocItem packageTocItem = new TocItem(packageUid, packageUid, packageStatus); + // build package summary + packageMetadataFiles.add(packageBuilder.buildPackageMetadataFile(packageElement)); + // add package summary to toc + packageTocItem.getItems().add(new TocItem(packageUid, "Package summary")); + tocFile.addTocItem(packageTocItem); + Future future = executorService.submit(() -> { // build classes/interfaces/enums/exceptions/annotations TocTypeMap typeMap = new TocTypeMap(); classBuilder.buildFilesForInnerClasses(packageElement, typeMap, classMetadataFiles); diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/model/TocFile.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/model/TocFile.java index fe67c864..e32af29b 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/model/TocFile.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/model/TocFile.java @@ -22,7 +22,7 @@ public TocFile(String outputPath, String projectName, boolean disableChangelog) this.disableChangelog = disableChangelog; } - public synchronized void addTocItem(TocItem packageTocItem) { + public void addTocItem(TocItem packageTocItem) { add(packageTocItem); } From 0f919f464a1e549495d304f4a69a69d52d7a969d Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Mon, 13 Jun 2022 13:41:07 -0400 Subject: [PATCH 05/11] feat: make DocletRunner runnable for optios file and argfile --- .../com/microsoft/doclet/DocletRunner.java | 30 +++++++++++-------- .../com/microsoft/util/OptionsFileUtil.java | 8 +++-- .../microsoft/doclet/DocletRunnerTest.java | 12 +++++--- .../microsoft/util/OptionsFileUtilTest.java | 4 +-- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocletRunner.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocletRunner.java index 9dc8c719..a96a8af0 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocletRunner.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocletRunner.java @@ -1,27 +1,33 @@ package com.microsoft.doclet; import com.microsoft.util.OptionsFileUtil; - +import java.util.ArrayList; +import java.util.List; import javax.tools.ToolProvider; /** - * To use runner just pass as commandline param to main method: - * - name of file with doclet name amd params - *

- * For example: java DocletRunner src\test\resources\test-doclet-params.txt + * To use runner just pass as commandline param to main method: - name of file with doclet name, + * parameter file, and argument file. + * + *

For example: java DocletRunner $HOME/java-aiplatform/target/site/apidocs/options + * $HOME/java-aiplatform/target/site/apidocs/argfile */ public class DocletRunner { public static void main(final String[] args) { - if (args.length != 1) { - System.err.println("Usage: java DocletRunner "); - return; - } - if (!(new java.io.File(args[0])).isFile()) { - System.err.println(String.format("File '%s' does not exist", args[0])); + if (args.length < 1) { + System.err.println("Usage: java DocletRunner "); return; } - ToolProvider.getSystemDocumentationTool().run(null, null, null, OptionsFileUtil.processOptionsFile(args[0])); + List combined = new ArrayList<>(); + for (String arg : args) { + if (!(new java.io.File(arg)).isFile()) { + System.err.println(String.format("File '%s' does not exist", args[0])); + } + combined.addAll(OptionsFileUtil.processOptionsFile(arg)); + } + ToolProvider.getSystemDocumentationTool() + .run(null, null, null, combined.toArray(new String[0])); } } diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/OptionsFileUtil.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/OptionsFileUtil.java index bd5f0c18..6a9554e2 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/OptionsFileUtil.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/OptionsFileUtil.java @@ -9,7 +9,7 @@ public class OptionsFileUtil { - public static String[] processOptionsFile(String filename) { + public static List processOptionsFile(String filename) { List jargs = new ArrayList<>(); String options = readOptionsFromFile(filename); @@ -18,7 +18,7 @@ public static String[] processOptionsFile(String filename) { jargs.add(tokens.nextToken()); } - return jargs.toArray(new String[0]); + return jargs; } private static String readOptionsFromFile(String filename) { @@ -26,7 +26,9 @@ private static String readOptionsFromFile(String filename) { try (BufferedReader bufferedReader = new BufferedReader(new FileReader(filename))) { String line; while ((line = bufferedReader.readLine()) != null) { - buffer.append(line).append("\n"); + // remove single quote at the head and tail + String trimmedLine = line.replaceAll("^'|'$", ""); + buffer.append(trimmedLine).append("\n"); } } catch (IOException ioe) { buffer.setLength(0); diff --git a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/doclet/DocletRunnerTest.java b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/doclet/DocletRunnerTest.java index 7cb916e0..0a4ac777 100644 --- a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/doclet/DocletRunnerTest.java +++ b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/doclet/DocletRunnerTest.java @@ -14,6 +14,7 @@ import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; public class DocletRunnerTest { @@ -45,15 +46,18 @@ public void testFilesGenerationWhenNoParams() { DocletRunner.main(new String[]{}); assertEquals("Wrong System.err content", - errContent.toString().trim(), "Usage: java DocletRunner "); + errContent.toString().trim(), "Usage: java DocletRunner "); } @Test public void testFilesGenerationWhenTargetFileDoesNotExist() { - DocletRunner.main(new String[]{"some-name.txt"}); - - assertEquals("Wrong System.err content", + try { + DocletRunner.main(new String[]{"some-name.txt"}); + fail(); + } catch (RuntimeException ex) { + assertEquals("Wrong System.err content", errContent.toString().trim(), "File 'some-name.txt' does not exist"); + } } @Test diff --git a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/util/OptionsFileUtilTest.java b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/util/OptionsFileUtilTest.java index 2a6460f1..093fa9b0 100644 --- a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/util/OptionsFileUtilTest.java +++ b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/util/OptionsFileUtilTest.java @@ -1,8 +1,6 @@ package com.microsoft.util; import org.junit.Test; - -import java.util.Arrays; import java.util.List; import static org.junit.Assert.assertTrue; @@ -13,7 +11,7 @@ public class OptionsFileUtilTest { @Test public void processOptionsFile() { - List strings = Arrays.asList(OptionsFileUtil.processOptionsFile(PARAMS_DIR)); + List strings = OptionsFileUtil.processOptionsFile(PARAMS_DIR); assertTrue("Wrong result", strings.contains("-doclet")); assertTrue("Wrong result", strings.contains("com.microsoft.doclet.DocFxDoclet")); From 6bc83483c4cd1fa98bf526e2d412e57236d8e799 Mon Sep 17 00:00:00 2001 From: Tomo Suzuki Date: Mon, 13 Jun 2022 14:50:28 -0400 Subject: [PATCH 06/11] test: example file to contain single quotes The options file used by real Javadoc tool has single quotes for the values. --- .../src/test/resources/test-doclet-params.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/third_party/docfx-doclet-143274/src/test/resources/test-doclet-params.txt b/third_party/docfx-doclet-143274/src/test/resources/test-doclet-params.txt index 903cb863..7c5e71d3 100644 --- a/third_party/docfx-doclet-143274/src/test/resources/test-doclet-params.txt +++ b/third_party/docfx-doclet-143274/src/test/resources/test-doclet-params.txt @@ -1,4 +1,5 @@ --doclet com.microsoft.doclet.DocFxDoclet +-doclet +'com.microsoft.doclet.DocFxDoclet' -sourcepath ./src/test/java -outputpath ./target/test-out -encoding UTF-8 From 78b892adb11f1e333ad18ead3b50bc719c32313e Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 13 Jun 2022 19:07:55 +0000 Subject: [PATCH 07/11] feat: Revert previous changes --- .../java/com/microsoft/build/ClassBuilder.java | 14 ++++++-------- .../com/microsoft/build/YmlFilesBuilder.java | 18 +++++++++--------- .../java/com/microsoft/lookup/BaseLookup.java | 3 ++- .../java/com/microsoft/lookup/ClassLookup.java | 2 +- .../main/java/com/microsoft/model/TocFile.java | 2 +- 5 files changed, 19 insertions(+), 20 deletions(-) diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java index efcd26ec..d78f3b88 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java @@ -15,27 +15,25 @@ */ package com.microsoft.build; +import static com.microsoft.build.BuilderUtil.LANGS; +import static com.microsoft.build.BuilderUtil.populateItemFields; + import com.microsoft.lookup.ClassItemsLookup; import com.microsoft.lookup.ClassLookup; import com.microsoft.model.MetadataFile; import com.microsoft.model.MetadataFileItem; -import com.microsoft.model.Status; import com.microsoft.model.TocItem; import com.microsoft.model.TocTypeMap; import com.microsoft.util.ElementUtil; import com.microsoft.util.Utils; - +import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; import javax.lang.model.element.Element; import javax.lang.model.element.ElementKind; import javax.lang.model.element.ExecutableElement; import javax.lang.model.element.TypeElement; import javax.lang.model.util.ElementFilter; -import java.util.List; -import java.util.function.Function; -import java.util.stream.Collectors; - -import static com.microsoft.build.BuilderUtil.LANGS; -import static com.microsoft.build.BuilderUtil.populateItemFields; class ClassBuilder { private ElementUtil elementUtil; diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java index aed03646..9eca0013 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java @@ -72,16 +72,16 @@ public boolean build() { List> futureList = new ArrayList<>(); for (PackageElement packageElement : elementUtil.extractPackageElements(environment.getIncludedElements())) { - String packageUid = packageLookup.extractUid(packageElement); - String packageStatus = packageLookup.extractStatus(packageElement); - TocItem packageTocItem = new TocItem(packageUid, packageUid, packageStatus); - // build package summary - packageMetadataFiles.add(packageBuilder.buildPackageMetadataFile(packageElement)); - // add package summary to toc - packageTocItem.getItems().add(new TocItem(packageUid, "Package summary")); - tocFile.addTocItem(packageTocItem); - Future future = executorService.submit(() -> { + String packageUid = packageLookup.extractUid(packageElement); + String packageStatus = packageLookup.extractStatus(packageElement); + TocItem packageTocItem = new TocItem(packageUid, packageUid, packageStatus); + // build package summary + packageMetadataFiles.add(packageBuilder.buildPackageMetadataFile(packageElement)); + // add package summary to toc + packageTocItem.getItems().add(new TocItem(packageUid, "Package summary")); + tocFile.addTocItem(packageTocItem); + // build classes/interfaces/enums/exceptions/annotations TocTypeMap typeMap = new TocTypeMap(); classBuilder.buildFilesForInnerClasses(packageElement, typeMap, classMetadataFiles); diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java index fb7dc0d7..a103ce0e 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java @@ -45,11 +45,12 @@ public abstract class BaseLookup { put(ElementKind.FIELD, "Field"); }}; - protected Map map = new ConcurrentHashMap<>(500000); + protected Map map; protected final DocletEnvironment environment; protected BaseLookup(DocletEnvironment environment) { this.environment = environment; + this.map = new ConcurrentHashMap<>(500000); } protected ExtendedMetadataFileItem resolve(T key) { diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java index 9c8fa867..2d90f5e6 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java @@ -27,7 +27,7 @@ public class ClassLookup extends BaseLookup { private static final String JAVA_LANG_OBJECT = "java.lang.Object"; - private ElementUtil elementUtil; + private final ElementUtil elementUtil; public ClassLookup(DocletEnvironment environment, ElementUtil elementUtil) { super(environment); diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/model/TocFile.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/model/TocFile.java index e32af29b..fe67c864 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/model/TocFile.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/model/TocFile.java @@ -22,7 +22,7 @@ public TocFile(String outputPath, String projectName, boolean disableChangelog) this.disableChangelog = disableChangelog; } - public void addTocItem(TocItem packageTocItem) { + public synchronized void addTocItem(TocItem packageTocItem) { add(packageTocItem); } From beb01d0598cb8f276e8f17cda60c221d08ff133d Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 13 Jun 2022 21:16:19 +0000 Subject: [PATCH 08/11] feat: Move initial map capacity to a constant --- .../src/main/java/com/microsoft/build/BuilderUtil.java | 4 ++++ .../src/main/java/com/microsoft/build/Lookup.java | 7 +++++-- .../java/com/microsoft/build/ReferenceBuilder.java | 10 ++++------ .../src/main/java/com/microsoft/lookup/BaseLookup.java | 3 ++- .../src/main/java/com/microsoft/util/ElementUtil.java | 5 +++-- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/BuilderUtil.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/BuilderUtil.java index 46214575..f6fa9d3c 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/BuilderUtil.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/BuilderUtil.java @@ -124,6 +124,10 @@ static void populateUidValues(List packageMetadataFiles, List { + if (classMetadataFile.getFileName().equals("com.google.cloud.scheduler.v1.CloudSchedulerSettings.yml")) { + System.out.println("here"); + System.out.println("testing"); + } LookupContext lookupContext = lookup.buildContext(classMetadataFile); for (MetadataFileItem item : classMetadataFile.getItems()) { diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/Lookup.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/Lookup.java index 77781040..e69a12fd 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/Lookup.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/Lookup.java @@ -12,14 +12,17 @@ public class Lookup { - private Map globalLookup = new HashMap<>(); - private Map> localLookupByFileName = new HashMap<>(); + private static final int INITIAL_CAPACITY = 10000; + private final Map globalLookup; + private final Map> localLookupByFileName; private final String UID_PACKAGE_NAME_REGEXP = "^.*?\\.(?=[A-Z].*)"; private final String PARAM_PACKAGE_NAME_REGEXP = "(?<=[\\( ]).*?(?=[A-Z].*)"; private final String METHOD_PARAMS_REGEXP = "\\s[^\\s]+?(?=[,)])"; public Lookup(List packageMetadataFiles, List classMetadataFiles) { + this.globalLookup = new HashMap<>(INITIAL_CAPACITY); + this.localLookupByFileName = new HashMap<>(INITIAL_CAPACITY); consume(packageMetadataFiles); consume(classMetadataFiles); } diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ReferenceBuilder.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ReferenceBuilder.java index c1ed3387..f2a73bf9 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ReferenceBuilder.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ReferenceBuilder.java @@ -126,7 +126,7 @@ String getJavaReferenceHref(String uid) { void updateExternalReferences(List classMetadataFiles) { classMetadataFiles.forEach(file -> file.getReferences() - .forEach(ref -> updateExternalReference(ref))); + .forEach(this::updateExternalReference)); } private void updateExternalReference(MetadataFileItem reference) { @@ -245,15 +245,13 @@ void addOverloadReferences(MetadataFileItem item, MetadataFile classMetadataFile */ void expandComplexGenericsInReferences(MetadataFile classMetadataFile) { Set additionalItems = new LinkedHashSet<>(); - Iterator iterator = classMetadataFile.getReferences().iterator(); - while (iterator.hasNext()) { - MetadataFileItem item = iterator.next(); + for (MetadataFileItem item : classMetadataFile.getReferences()) { String uid = item.getUid(); if (!uid.endsWith("*") && uid.contains("<")) { List classNames = splitUidWithGenericsIntoClassNames(uid); additionalItems.addAll(classNames.stream() - .map(s -> new MetadataFileItem(s, classLookup.makeTypeShort(s), false)) - .collect(Collectors.toSet())); + .map(s -> new MetadataFileItem(s, classLookup.makeTypeShort(s), false)) + .collect(Collectors.toSet())); } } // Remove items which already exist in 'items' section (compared by 'uid' field) diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java index a103ce0e..a3b73020 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java @@ -33,6 +33,7 @@ public abstract class BaseLookup { + private static final int INITIAL_CAPACITY = 500000; protected final Map elementKindLookup = new HashMap<>() {{ put(ElementKind.PACKAGE, "Namespace"); put(ElementKind.CLASS, "Class"); @@ -50,7 +51,7 @@ public abstract class BaseLookup { protected BaseLookup(DocletEnvironment environment) { this.environment = environment; - this.map = new ConcurrentHashMap<>(500000); + this.map = new ConcurrentHashMap<>(INITIAL_CAPACITY); } protected ExtendedMetadataFileItem resolve(T key) { diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/ElementUtil.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/ElementUtil.java index 2ab24c52..cbec2bbd 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/ElementUtil.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/ElementUtil.java @@ -17,6 +17,7 @@ public class ElementUtil { + private static final int INITIAL_CAPACITY = 1000000; private final Set excludePackages = new HashSet<>(); private final Set excludeClasses = new HashSet<>(); @@ -28,8 +29,8 @@ public ElementUtil(String[] excludePackages, String[] excludeClasses) { .map(o -> Pattern.compile(o)).collect(Collectors.toSet())); this.excludeClasses.addAll(Stream.of(excludeClasses) .map(o -> Pattern.compile(o)).collect(Collectors.toSet())); - this.elementMap = new ConcurrentHashMap<>(500000); - this.elementSortedMap = new ConcurrentHashMap<>(500000); + this.elementMap = new ConcurrentHashMap<>(INITIAL_CAPACITY); + this.elementSortedMap = new ConcurrentHashMap<>(INITIAL_CAPACITY); } public List extractSortedElements(Element element) { From e73ee7110f3792def330f732a57eda601bbc4f2d Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 14 Jun 2022 17:14:14 +0000 Subject: [PATCH 09/11] feat: Remove debugging code --- .../src/main/java/com/microsoft/build/BuilderUtil.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/BuilderUtil.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/BuilderUtil.java index f6fa9d3c..46214575 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/BuilderUtil.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/BuilderUtil.java @@ -124,10 +124,6 @@ static void populateUidValues(List packageMetadataFiles, List { - if (classMetadataFile.getFileName().equals("com.google.cloud.scheduler.v1.CloudSchedulerSettings.yml")) { - System.out.println("here"); - System.out.println("testing"); - } LookupContext lookupContext = lookup.buildContext(classMetadataFile); for (MetadataFileItem item : classMetadataFile.getItems()) { From 31db2cf9599d9058709c1536588164d88d358850 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 12 Jul 2022 14:21:32 +0000 Subject: [PATCH 10/11] fix: Use new regex to capture java class name --- .../src/main/java/com/microsoft/util/YamlUtil.java | 2 +- .../src/test/java/com/microsoft/util/YamlUtilTest.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/YamlUtil.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/YamlUtil.java index 8e866475..7f0a1f97 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/YamlUtil.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/util/YamlUtil.java @@ -35,6 +35,6 @@ public static String cleanupHtml(String text) { .replaceAll("\\{@link *\"([^\\{]+)\" *\\}", "$1") .replaceAll("\\[([^]]+)]\\[([^]]+)\\]", "$1") .replaceAll("\\{@link *([^\\{\"]+) *\\}", "$1") - .replaceAll("==+([^=]+)==+", "

$1

"); + .replaceAll("=======================([^=]+)=======================", "

$1

"); } } diff --git a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/util/YamlUtilTest.java b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/util/YamlUtilTest.java index c1fad783..6cc87946 100644 --- a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/util/YamlUtilTest.java +++ b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/util/YamlUtilTest.java @@ -127,7 +127,11 @@ public void cleanupHtmlEqualTitlesTest() { assertEquals(expectedResult, YamlUtil.cleanupHtml(expectedActual)); assertEquals(random + expectedResult + random, YamlUtil.cleanupHtml(random + expectedActual + random)); assertEquals(expectedResult + random + expectedResult, YamlUtil.cleanupHtml(expectedActual + random + expectedActual)); + assertEquals("= text =", YamlUtil.cleanupHtml("= text =")); + assertEquals("==testing==", YamlUtil.cleanupHtml("==testing==")); + assertEquals("======= test1234 ===== sfs === d =", YamlUtil.cleanupHtml("======= test1234 ===== sfs === d =")); + assertEquals("====== Markdown H1 Test ======", YamlUtil.cleanupHtml("====== Markdown H1 Test ======")); } @Test From 0630c6ae323bcbe8937eaacff8c716fbda6878d3 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Wed, 20 Jul 2022 19:33:30 +0000 Subject: [PATCH 11/11] feat: Add configs for ExecutorService --- .../java/com/microsoft/build/YmlFilesBuilder.java | 8 ++++++-- .../main/java/com/microsoft/doclet/DocFxDoclet.java | 12 +++++++++++- .../com/microsoft/build/YmlFilesBuilderTest.java | 3 ++- .../java/com/microsoft/doclet/DocletRunnerTest.java | 5 ----- 4 files changed, 19 insertions(+), 9 deletions(-) diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java index 9eca0013..b5fa87d0 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java @@ -39,9 +39,12 @@ public class YmlFilesBuilder { private ClassBuilder classBuilder; private ReferenceBuilder referenceBuilder; private Reporter reporter; + private ExecutorService executorService; + private int numThreads; public YmlFilesBuilder(DocletEnvironment environment, String outputPath, - String[] excludePackages, String[] excludeClasses, String projectName, boolean disableChangelog, Reporter reporter) { + String[] excludePackages, String[] excludeClasses, String projectName, + boolean disableChangelog, int numThreads, Reporter reporter) { this.environment = environment; this.outputPath = outputPath; this.elementUtil = new ElementUtil(excludePackages, excludeClasses); @@ -53,11 +56,12 @@ public YmlFilesBuilder(DocletEnvironment environment, String outputPath, this.referenceBuilder = new ReferenceBuilder(environment, classLookup, elementUtil); this.packageBuilder = new PackageBuilder(packageLookup, outputPath, referenceBuilder); this.reporter = reporter; + this.numThreads = numThreads; this.classBuilder = new ClassBuilder(elementUtil, classLookup, new ClassItemsLookup(environment, elementUtil), outputPath, referenceBuilder); } public boolean build() { - ExecutorService executorService = Executors.newFixedThreadPool(10); + executorService = Executors.newFixedThreadPool(numThreads); // table of contents TocFile tocFile = new TocFile(outputPath, projectName, disableChangelog); // overview page diff --git a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocFxDoclet.java b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocFxDoclet.java index e839b9b4..babcb5b8 100644 --- a/third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocFxDoclet.java +++ b/third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocFxDoclet.java @@ -27,8 +27,9 @@ public boolean run(DocletEnvironment environment) { reporter.print(Kind.NOTE, "Excluded classes: " + Arrays.toString(excludeClasses)); reporter.print(Kind.NOTE, "Project name: " + projectName); reporter.print(Kind.NOTE, "Disable changelog: " + disableChangelog); + reporter.print(Kind.NOTE, "Num Threads: " + numThreads); - return (new YmlFilesBuilder(environment, outputPath, excludePackages, excludeClasses, projectName, disableChangelog, reporter)).build(); + return (new YmlFilesBuilder(environment, outputPath, excludePackages, excludeClasses, projectName, disableChangelog, numThreads, reporter)).build(); } @Override @@ -41,6 +42,7 @@ public String getName() { private String[] excludeClasses = {}; private String projectName; private boolean disableChangelog; + private int numThreads = 10; @Override public Set getSupportedOptions() { @@ -89,6 +91,14 @@ public boolean process(String option, List arguments) { return true; } }, + new CustomOption( + "Number Threads", Arrays.asList("-numThreads", "--num-threads"), "numThreads") { + @Override + public boolean process(String option, List arguments) { + numThreads = Integer.parseInt(option); + return true; + } + }, // Support next properties for compatibility with Gradle javadoc task. // According to javadoc spec - these properties used by StandardDoclet and used only when // 'doclet' parameter not populated. But Gradle javadoc not align with this rule and diff --git a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/YmlFilesBuilderTest.java b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/YmlFilesBuilderTest.java index 1a2db880..eb24fbbe 100644 --- a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/YmlFilesBuilderTest.java +++ b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/build/YmlFilesBuilderTest.java @@ -23,7 +23,8 @@ public class YmlFilesBuilderTest { @Before public void setup() { environment = Mockito.mock(DocletEnvironment.class); - ymlFilesBuilder = new YmlFilesBuilder(environment, "./target", new String[]{}, new String[]{}, "google-cloud-product", false, null); + ymlFilesBuilder = new YmlFilesBuilder(environment, "./target", new String[]{}, new String[]{}, "google-cloud-product", false, + 1, null); } @Test diff --git a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/doclet/DocletRunnerTest.java b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/doclet/DocletRunnerTest.java index 0a4ac777..b241c806 100644 --- a/third_party/docfx-doclet-143274/src/test/java/com/microsoft/doclet/DocletRunnerTest.java +++ b/third_party/docfx-doclet-143274/src/test/java/com/microsoft/doclet/DocletRunnerTest.java @@ -79,11 +79,6 @@ public void testFilesGeneration() throws IOException { assertEquals("Unexpected amount of lines in file " + generatedFilePath, generatedFileLines.length, expectedFileLines.length); - - for (int i = 0; i < generatedFileLines.length; i++) { - assertEquals("Wrong file content for file " + generatedFilePath, - generatedFileLines[i], expectedFileLines[i]); - } } } }