Changelog script. Read dependencies, support N/A (#5181)

Fixes
https://youtrack.jetbrains.com/issue/CMP-7203/Changelog-script.-Generate-Dependencies-section
Fixes
https://youtrack.jetbrains.com/issue/CMP-7137/Changelog-script-small-fixes

- Read dependencies from git repos
- Support N/A
- Supports any prefix. For example `- (experimental) Change`
- Try to fix the line start

Can be read commit by commit.

.github/PULL_REQUEST_TEMPLATE.md

@@ -15,16 +15,18 @@ This should be tested by QA
 
 ## Release Notes
 <!--
-Optional, if omitted - won't be included in the changelog
+If we definitely shouldn't add Release Notes, add only N/A.
 
-Sections:
+Or enumerate sections, subsections and all changes.
+
+Possible sections:
 - Highlights
 - Known Issues
 - Breaking Changes
 - Features
 - Fixes
 
-Subsections:
+Possible subsections:
 - Multiple Platforms
 - iOS
 - Desktop

tools/changelog.main.kts

@@ -28,13 +28,15 @@
 @file:DependsOn("com.google.code.gson:gson:2.10.1")
 
 import com.google.gson.Gson
-import java.io.File
 import java.io.IOException
+import java.lang.ProcessBuilder.Redirect
 import java.net.URL
+import java.net.URLEncoder
+import java.nio.charset.StandardCharsets.UTF_8
 import java.time.LocalDate
 import java.time.format.DateTimeFormatter
 import java.util.*
-import java.util.concurrent.TimeUnit
+import kotlin.text.substringAfterLast
 
 //region ========================================== CONSTANTS =========================================
 
@@ -73,7 +75,6 @@ val argsKeyToValue = args
     .associate { it.substringBefore("=") to it.substringAfter("=") }
 
 val versionCommit = argsKeyless.getOrNull(0) ?: "HEAD"
-val versionName = argsKeyless.getOrNull(1) ?: versionCommit
 val token = argsKeyToValue["token"]
 
 println("Note. The script supports optional arguments: kotlin changelog.main.kts [versionCommit] [versionName] [token=githubToken]")
@@ -82,6 +83,34 @@ if (token == null) {
 }
 println()
 
+val androidxLibToVersion = androidxLibToVersion(versionCommit)
+val androidxLibToRedirectingVersion = androidxLibToRedirectingVersion(versionCommit)
+
+fun formatAndroidxLibVersion(libName: String) =
+    androidxLibToVersion[libName] ?: "PLACEHOLDER".also {
+        println("Can't find $libName version. Using PLACEHOLDER")
+    }
+
+fun formatAndroidxLibRedirectingVersion(libName: String) =
+    androidxLibToRedirectingVersion[libName] ?: "PLACEHOLDER".also {
+        println("Can't find $libName redirecting version. Using PLACEHOLDER")
+    }
+
+val versionCompose = formatAndroidxLibVersion("COMPOSE")
+val versionComposeMaterial3Adaptive = formatAndroidxLibVersion("COMPOSE_MATERIAL3_ADAPTIVE")
+val versionLifecycle = formatAndroidxLibVersion("LIFECYCLE")
+val versionNavigation = formatAndroidxLibVersion("NAVIGATION")
+
+val versionRedirectingCompose = formatAndroidxLibRedirectingVersion("compose")
+val versionRedirectingComposeFoundation = formatAndroidxLibRedirectingVersion("compose.foundation")
+val versionRedirectingComposeMaterial = formatAndroidxLibRedirectingVersion("compose.material")
+val versionRedirectingComposeMaterial3 = formatAndroidxLibRedirectingVersion("compose.material3")
+val versionRedirectingComposeMaterial3Adaptive = formatAndroidxLibRedirectingVersion("compose.material3.adaptive")
+val versionRedirectingLifecycle = formatAndroidxLibRedirectingVersion("lifecycle")
+val versionRedirectingNavigation = formatAndroidxLibRedirectingVersion("navigation")
+
+val versionName = versionCompose
+
 val currentChangelog = changelogFile.readText()
 val previousChangelog =
     if (currentChangelog.startsWith("# $versionName ")) {
@@ -93,9 +122,10 @@ val previousChangelog =
 
 val previousVersion = previousChangelog.substringAfter("# ").substringBefore(" (")
 
+println()
 println("Generating changelog between $previousVersion and $versionName")
 
-val newChangelog = getChangelog("v$previousVersion", versionCommit, previousVersion, versionName)
+val newChangelog = getChangelog("v$previousVersion", versionCommit, previousVersion)
 
 changelogFile.writeText(
     newChangelog + previousChangelog
@@ -105,12 +135,12 @@ println()
 println("CHANGELOG.md changed")
 
 
-fun getChangelog(firstCommit: String, lastCommit: String, firstVersion: String, lastVersion: String): String {
+fun getChangelog(firstCommit: String, lastCommit: String, firstVersion: String): String {
     val entries = entriesForRepo("JetBrains/compose-multiplatform-core", firstCommit, lastCommit) +
             entriesForRepo("JetBrains/compose-multiplatform", firstCommit, lastCommit)
 
     return buildString {
-        appendLine("# $lastVersion (${currentChangelogDate()})")
+        appendLine("# $versionName (${currentChangelogDate()})")
 
         appendLine()
         appendLine("_Changes since ${firstVersion}_")
@@ -140,16 +170,16 @@ fun getChangelog(firstCommit: String, lastCommit: String, firstVersion: String,
             """
                 ## Dependencies
         
-                - Gradle Plugin `org.jetbrains.compose`, version `$lastVersion`. Based on Jetpack Compose libraries:
+                - Gradle Plugin `org.jetbrains.compose`, version `$versionCompose`. Based on Jetpack Compose libraries:
-                  - [Runtime REDIRECT_PLACEHOLDER](https://developer.android.com/jetpack/androidx/releases/compose-runtime#REDIRECT_PLACEHOLDER)
+                  - [Runtime $versionRedirectingCompose](https://developer.android.com/jetpack/androidx/releases/compose-runtime#$versionRedirectingCompose)
-                  - [UI REDIRECT_PLACEHOLDER](https://developer.android.com/jetpack/androidx/releases/compose-ui#REDIRECT_PLACEHOLDER)
+                  - [UI $versionRedirectingCompose](https://developer.android.com/jetpack/androidx/releases/compose-ui#$versionRedirectingCompose)
-                  - [Foundation REDIRECT_PLACEHOLDER](https://developer.android.com/jetpack/androidx/releases/compose-foundation#REDIRECT_PLACEHOLDER)
+                  - [Foundation $versionRedirectingComposeFoundation](https://developer.android.com/jetpack/androidx/releases/compose-foundation#$versionRedirectingComposeFoundation)
-                  - [Material REDIRECT_PLACEHOLDER](https://developer.android.com/jetpack/androidx/releases/compose-material#REDIRECT_PLACEHOLDER)
+                  - [Material $versionRedirectingComposeMaterial](https://developer.android.com/jetpack/androidx/releases/compose-material#$versionRedirectingComposeMaterial)
-                  - [Material3 REDIRECT_PLACEHOLDER](https://developer.android.com/jetpack/androidx/releases/compose-material3#REDIRECT_PLACEHOLDER)
+                  - [Material3 $versionRedirectingComposeMaterial3](https://developer.android.com/jetpack/androidx/releases/compose-material3#$versionRedirectingComposeMaterial3)
-        
+
-                - Lifecycle libraries `org.jetbrains.androidx.lifecycle:lifecycle-*:RELEASE_PLACEHOLDER`. Based on [Jetpack Lifecycle REDIRECT_PLACEHOLDER](https://developer.android.com/jetpack/androidx/releases/lifecycle#REDIRECT_PLACEHOLDER)
+                - Lifecycle libraries `org.jetbrains.androidx.lifecycle:lifecycle-*:$versionLifecycle`. Based on [Jetpack Lifecycle $versionRedirectingLifecycle](https://developer.android.com/jetpack/androidx/releases/lifecycle#$versionRedirectingLifecycle)
-                - Navigation libraries `org.jetbrains.androidx.navigation:navigation-*:RELEASE_PLACEHOLDER`. Based on [Jetpack Navigation REDIRECT_PLACEHOLDER](https://developer.android.com/jetpack/androidx/releases/navigation#REDIRECT_PLACEHOLDER)
+                - Navigation libraries `org.jetbrains.androidx.navigation:navigation-*:$versionNavigation`. Based on [Jetpack Navigation $versionRedirectingNavigation](https://developer.android.com/jetpack/androidx/releases/navigation#$versionRedirectingNavigation)
-                - Material3 Adaptive libraries `org.jetbrains.compose.material3.adaptive:adaptive*:RELEASE_PLACEHOLDER`. Based on [Jetpack Material3 Adaptive REDIRECT_PLACEHOLDER](https://developer.android.com/jetpack/androidx/releases/compose-material3-adaptive#REDIRECT_PLACEHOLDER)
+                - Material3 Adaptive libraries `org.jetbrains.compose.material3.adaptive:adaptive*:$versionComposeMaterial3Adaptive`. Based on [Jetpack Material3 Adaptive $versionRedirectingComposeMaterial3Adaptive](https://developer.android.com/jetpack/androidx/releases/compose-material3-adaptive#$versionRedirectingComposeMaterial3Adaptive)
         
                 ---
             """.trimIndent()
@@ -188,13 +218,26 @@ fun currentChangelogDate() = LocalDate.now().format(DateTimeFormatter.ofPattern(
  * - [A new approach to implementation of `platformLayers`](link). Now extra layers (such as Dialogs and Popups) drawing is merged into a single screen size canvas.
  */
 fun ChangelogEntry.format(): String {
+    return try {
+        tryFormat()
+    } catch (e: Exception) {
+        throw RuntimeException("Formatting error of ChangelogEntry. Message:\n$message", e)
+    }
+}
+
+fun ChangelogEntry.tryFormat(): String {
     return if (link != null) {
+        val prefixRegex = "^[-\\s]*"   // "- "
+        val tagRegex1 = "\\(.*\\)\\s*" // "(something) "
+        val tagRegex2 = "\\[.*\\]\\s*" // "[something] "
+        val tagRegex3 = "_.*_\\s*"     // "_something_ "
         val linkStartIndex = maxOf(
-            message.indexOfFirst { !it.isWhitespace() && it != '-' }.ifNegative { 0 },
+            message.endIndexOfFirstGroup(Regex("($prefixRegex).*"))?.plus(1) ?: 0,
-            message.endIndexOf("_(prerelease fix)_ ").ifNegative { 0 },
+            message.endIndexOfFirstGroup(Regex("($prefixRegex$tagRegex1).*"))?.plus(1) ?: 0,
-            message.endIndexOf("(prerelease fix) ").ifNegative { 0 },
+            message.endIndexOfFirstGroup(Regex("($prefixRegex$tagRegex2).*"))?.plus(1) ?: 0,
+            message.endIndexOfFirstGroup(Regex("($prefixRegex$tagRegex3).*"))?.plus(1) ?: 0,
         )
-        val linkLastIndex = message.indexOfAny(listOf(". ", " (")).ifNegative { message.length }
+        val linkLastIndex = message.indexOfAny(listOf(". ", " ("), linkStartIndex).ifNegative { message.length }
 
         val beforeLink = message.substring(0, linkStartIndex)
         val inLink = message.substring(linkStartIndex, linkLastIndex).removeLinks()
@@ -208,13 +251,8 @@ fun ChangelogEntry.format(): String {
 
 fun Int.ifNegative(value: () -> Int): Int = if (this < 0) value() else this
 
-fun String.endIndexOf(value: String): Int = indexOf(value).let {
+fun String.endIndexOfFirstGroup(regex: Regex): Int? =
-    if (it >= 0) {
+    regex.find(this)?.groups?.toList()?.getOrNull(1)?.range?.endInclusive
-        it + value.length
-    } else {
-        it
-    }
-}
 
 /**
  * Converts:
@@ -243,9 +281,13 @@ fun GitHubPullEntry.extractReleaseNotes(link: String): List<ChangelogEntry> {
         before?.trim()
     }
 
+    if (relNoteBody?.trim()?.lowercase() == "n/a") return emptyList()
+
     val list = mutableListOf<ChangelogEntry>()
     var section: String? = null
     var subsection: String? = null
+    var isFirstLine = true
+    var shouldPadLines = false
 
     for (line in relNoteBody.orEmpty().split("\n")) {
         // parse "### Section - Subsection"
@@ -253,17 +295,30 @@ fun GitHubPullEntry.extractReleaseNotes(link: String): List<ChangelogEntry> {
             val s = line.removePrefix("### ")
             section = s.substringBefore("-", "").trim().normalizeSectionName().ifEmpty { null }
             subsection = s.substringAfter("-", "").trim().normalizeSubsectionName().ifEmpty { null }
+            isFirstLine = true
+            shouldPadLines = false
         } else if (section != null && line.isNotBlank()) {
-            val isTopLevel = line.startsWith("-")
+            var lineFixed = line
-            val trimmedLine = line.trimEnd().removeSuffix(".")
+
+            if (isFirstLine && !lineFixed.startsWith("-")) {
+                lineFixed = "- $lineFixed"
+                shouldPadLines = true
+            }
+            if (!isFirstLine && shouldPadLines) {
+                lineFixed = "  $lineFixed"
+            }
+            lineFixed = lineFixed.trimEnd().removeSuffix(".")
+
+            val isTopLevel = lineFixed.startsWith("-")
             list.add(
                 ChangelogEntry(
-                    trimmedLine,
+                    lineFixed,
                     section,
                     subsection,
                     link.takeIf { isTopLevel }
                 )
             )
+            isFirstLine = false
         }
     }
 
@@ -277,7 +332,7 @@ fun GitHubPullEntry.extractReleaseNotes(link: String): List<ChangelogEntry> {
 fun entriesForRepo(repo: String, firstCommit: String, lastCommit: String): List<ChangelogEntry> {
     val pulls = (1..5)
         .flatMap {
-            request<Array<GitHubPullEntry>>("https://api.github.com/repos/$repo/pulls?state=closed&per_page=100&page=$it").toList()
+            requestJson<Array<GitHubPullEntry>>("https://api.github.com/repos/$repo/pulls?state=closed&per_page=100&page=$it").toList()
         }
 
     val pullNumberToPull = pulls.associateBy { it.number }
@@ -310,8 +365,7 @@ fun entriesForRepo(repo: String, firstCommit: String, lastCommit: String): List<
     fun fetchCommits(firsCommitSha: String, lastCommitSha: String): CommitsResult {
         lateinit var mergeBaseCommit: String
         val commits = fetchPagedUntilEmpty { page ->
-            val result =
+            val result = requestJson<GitHubCompareResponse>("https://api.github.com/repos/$repo/compare/$firsCommitSha...$lastCommitSha?per_page=1000&page=$page")
-                request<GitHubCompareResponse>("https://api.github.com/repos/$repo/compare/$firsCommitSha...$lastCommitSha?per_page=1000&page=$page")
             mergeBaseCommit = result.merge_base_commit.sha
             result.commits
         }
@@ -336,6 +390,55 @@ fun repoTitleAndNumberForCommit(commit: GitHubCompareResponse.CommitEntry): Pair
     return title to number
 }
 
+/**
+ * Extract redirecting versions from core repo, file gradle.properties
+ *
+ * Example
+ * https://raw.githubusercontent.com/JetBrains/compose-multiplatform-core/v1.8.0%2Bdev1966/gradle.properties
+ * artifactRedirecting.androidx.graphics.version=1.0.1
+ */
+fun androidxLibToRedirectingVersion(commit: String): Map<String, String> {
+    val gradleProperties = githubContentOf("JetBrains/compose-multiplatform-core", "gradle.properties", commit)
+    val regex = Regex("artifactRedirecting\\.androidx\\.(.*)\\.version=(.*)")
+    return regex.findAll(gradleProperties).associate { result ->
+        result.groupValues[1].trim() to result.groupValues[2].trim()
+    }
+}
+
+/**
+ * Extract versions from CI config, file .teamcity/compose/Library.kt
+ *
+ * Example
+ * https://jetbrains.team/p/ui/repositories/compose-teamcity-config/files/8f8408ccd05a9188895969b1fa0243050716baad/.teamcity/compose/Library.kt?tab=source&line=37&lines-count=1
+ * Library.CORE_BUNDLE -> "1.1.0-alpha01"
+ */
+fun androidxLibToVersion(commit: String): Map<String, String> {
+    val repo = "ssh://git@git.jetbrains.team/ui/compose-teamcity-config.git"
+    val file = ".teamcity/compose/Library.kt"
+    val libraryKt = spaceContentOf(repo, file, commit)
+
+    return if (libraryKt.isBlank()) {
+        println("Can't clone $repo to know library versions. Please register your ssh key in https://jetbrains.team/m/me/authentication?tab=GitKeys")
+        emptyMap()
+    } else {
+        val regex = Regex("Library\\.(.*)\\s*->\\s*\"(.*)\"")
+        return regex.findAll(libraryKt).associate { result ->
+            result.groupValues[1].trim() to result.groupValues[2].trim()
+        }
+    }
+}
+
+fun githubContentOf(repo: String, path: String, commit: String): String {
+    val commitEncoded = URLEncoder.encode(commit, UTF_8)
+    return requestPlain("https://raw.githubusercontent.com/$repo/$commitEncoded/$path")
+}
+
+fun spaceContentOf(repoUrl: String, path: String, tagName: String): String {
+    return pipeProcess("git archive --remote=$repoUrl $tagName $path")
+        .pipeTo("tar -xO $path")
+        .readText()
+}
+
 data class ChangelogEntry(
     val message: String,
     val section: String?,
@@ -362,37 +465,25 @@ data class GitHubPullEntry(val number: Int, val title: String, val body: String?
 }
 
 //region ========================================== UTILS =========================================
-
+fun pipeProcess(command: String) = ProcessBuilder(command.split(" "))
-// from https://stackoverflow.com/a/41495542
+    .redirectOutput(Redirect.PIPE)
-fun String.runCommand(workingDir: File = File(".")) {
+    .redirectError(Redirect.PIPE)
-    ProcessBuilder(*split(" ").toTypedArray())
+    .start()!!
-        .directory(workingDir)
+
-        .redirectOutput(ProcessBuilder.Redirect.INHERIT)
+fun Process.pipeTo(command: String): Process = pipeProcess(command).also {
-        .redirectError(ProcessBuilder.Redirect.INHERIT)
+    inputStream.use { input ->
-        .start()
+        it.outputStream.use { out ->
-        .waitFor(5, TimeUnit.MINUTES)
+            input.copyTo(out)
-}
+        }
-
-fun String.execCommand(workingDir: File = File(".")): String? {
-    try {
-        val parts = this.split("\\s".toRegex())
-        val proc = ProcessBuilder(*parts.toTypedArray())
-            .directory(workingDir)
-            .redirectOutput(ProcessBuilder.Redirect.PIPE)
-            .redirectError(ProcessBuilder.Redirect.PIPE)
-            .start()
-
-        proc.waitFor(60, TimeUnit.MINUTES)
-        return proc.inputStream.bufferedReader().readText()
-    } catch (e: IOException) {
-        e.printStackTrace()
-        return null
     }
 }
 
-inline fun <reified T> request(
+fun Process.readText(): String = inputStream.bufferedReader().use { it.readText() }
-    url: String
+
-): T = exponentialRetry {
+inline fun <reified T> requestJson(url: String): T =
+    Gson().fromJson(requestPlain(url), T::class.java)
+
+fun requestPlain(url: String): String = exponentialRetry {
     println("Request $url")
     val connection = URL(url).openConnection()
     connection.setRequestProperty("User-Agent", "Compose-Multiplatform-Script")
@@ -400,10 +491,7 @@ inline fun <reified T> request(
         connection.setRequestProperty("Authorization", "Bearer $token")
     }
     connection.getInputStream().use {
-        Gson().fromJson(
+        it.bufferedReader().readText()
-            it.bufferedReader(),
-            T::class.java
-        )
     }
 }