Process of rewriting build system... yet again

.gitignore

@@ -1,11 +1,8 @@
 build/
 source-apk/*.apk
 keystores/*.keystore
-tools/*
-!tools/smali-2.5.2.jar
-!tools/apktool_2.9.3.jar
-!tools/apksigner.jar
-!tools/d8.jar
 
 .idea
 *.iml
+
+__pycache__

.vscode/settings.json

@@ -0,0 +1,2 @@
+{
+}

build.py

@@ -1,234 +0,0 @@
-import sys
-import shutil
-import os
-from pathlib import Path
-import subprocess
-
-class BuildError(Exception):
-    def __init__(self):
-        super.__init__()
-
-def getDexName(smaliName):
-    if smaliName == 'smali':
-        return 'classes.dex'
-    elif smaliName.startswith('smali_classes'):
-        return 'classes' + smaliName[13:] + '.dex'
-    print("Not a smali name")
-    raise BuildError()
-
-class Task:
-    def clean():
-        shutil.rmtree('build')
-    
-    def extractApk():
-        if Path('build/extracted').exists():
-            return
-        Path('build').mkdir(parents=True, exist_ok=True)
-
-        apk = list(filter(lambda f: f.endswith('.apk'), os.listdir('source-apk')))
-        if len(apk) == 0:
-            print('Please place the APK inside the source-apk directory.')
-            raise BuildError()
-        elif len(apk) > 1:
-            print('More than one apk found in source-apk. Please remove all but one.')
-            raise BuildError()
-
-        # Extract the apk
-        print("Extracting APK...")
-        subprocess.run(['java', '-jar', './tools/apktool_2.9.3.jar', 'd', 'source-apk/' + apk[0], '-o', 'build/extracted'])
-
-    def copySmali():
-        Task.extractApk()
-
-        Path('build/smali').mkdir(parents=True, exist_ok=True)
-        # Copy smali classes that haven't been yet
-        for file in os.listdir('build/extracted'):
-            if file != 'smali' and not file.startswith('smali_classes'):
-                continue
-            
-            # Skip if already exists
-            if Path(f'build/smali/{file}').exists():
-                continue
-
-            print(f'Copying {file}...')
-            shutil.copytree(f'build/extracted/{file}', f'build/smali/{file}')
-
-        # Copy from src
-        print('Copying ykit smali...')
-        shutil.copytree('src/smali', 'build/smali', dirs_exist_ok=True)
-
-    def compileSmali():
-        Task.copySmali()
-
-        Path('build/dex').mkdir(parents=True, exist_ok=True)
-
-        def shouldCompileSmali(file, dex):
-            src = f'build/smali/{file}'
-            ykitsrc = f'src/smali/{file}'
-            dst = f'build/dex/{dex}'
-
-            # If dest doesn't exist, always build
-            if not Path(dst).exists():
-                return True
-
-            # Check if ykit src dir exists for it, if not then don't recompile
-            if not Path(ykitsrc).exists():
-                return False
-
-            # Get latest file timestamp in ykitsrc
-            files = list(Path(ykitsrc).rglob('*.smali'))
-            files = [os.path.getmtime(x) for x in files]
-            latest = max(files)
-
-            dsttime = os.path.getmtime(dst)
-
-            # If it is newer than the dest, then do recompile
-            if latest > dsttime:
-                return True
-
-            # Otherwise, don't
-            return False
-
-        for file in os.listdir('build/smali'):
-            dex = getDexName(file)
-
-            if not shouldCompileSmali(file, dex):
-                print(f"Skipping {file}...")
-                continue
-
-            print(f"Compiling {file}...")
-            subprocess.run(['java', '-jar', 'tools/smali-2.5.2.jar', 'a', f'build/smali/{file}', '-o', f'build/dex/{dex}'])
-
-    def genClassPath():
-        print("Generating java classpath from dex files...")
-        for file in os.listdir('build/dex'):
-            src = f'build/dex/{file}'
-            dst = f'build/jar/{file[:-4]}.jar'
-            srctime = os.path.getmtime(src)
-            dsttime = 0 if not Path(dst).exists() else os.path.getmtime(dst)
-
-            if srctime < dsttime:
-                print(f"Skipping {file}...")
-                continue
-
-            print(f"Dex2jar {file}...")
-            subprocess.run(['java', '-cp', 'tools/d2j/*', 'com.googlecode.dex2jar.tools.Dex2jarCmd', '-f', src, '-o', dst])
-
-    def compileJava():
-        Task.genClassPath()
-
-        sep = ';' if os.name == 'nt' else ':'
-        javaFiles = [str(x) for x in Path('src/java').rglob('*.java')]
-        classPath = sep.join([str(x) for x in Path('build/jar').rglob('*.jar')])
-
-        # Get latest java file timestamp
-        files = list(Path('src/java').rglob('*.java'))
-        files = [os.path.getmtime(x) for x in files]
-        latest = max(files)
-
-        # Get latest dex time or 0
-        latest2 = 0 if not Path('build/java_dex/classes.dex').exists() else os.path.getmtime('build/java_dex/classes.dex')
-
-        # If dex file is newer than source, no need to do anything
-        if latest2 > latest:
-            print("Skipping java...")
-            return
-
-        print("Compiling java...")
-        subprocess.run(['javac', '-cp', classPath, '-d', 'build/java_class', *javaFiles])
-
-        classFiles = [str(x) for x in Path('build/java_class').rglob('*.class')]
-
-        Path('build/java_dex').mkdir(parents=True, exist_ok=True)
-        subprocess.run(['java', '-cp', 'tools/d8.jar', 'com.android.tools.r8.D8', *classFiles, '--output', 'build/java_dex'])
-
-    def mergeResources():
-        Task.compileSmali()
-        Task.compileJava()
-
-        # Copy original resources
-        # on first run only
-        if not Path('build/merged').exists():
-            print('Copying extracted resources...')
-            shutil.copytree('build/extracted', 'build/merged', dirs_exist_ok=True, ignore=shutil.ignore_patterns('smali', 'smali_classes*'))
-
-        print("Copying ykit resources...")
-        shutil.copytree('src/resources', 'build/merged', dirs_exist_ok=True)
-
-        print("Copying compiled dex files...")
-        shutil.copytree('build/dex', 'build/merged', dirs_exist_ok=True)
-        shutil.copy('build/java_dex/classes.dex', 'build/merged/classes7.dex') # TODO: Unhardcode this
-
-    def buildApk():
-        Task.mergeResources()
-
-        debugFlag = ('-d',) if ('d' in ''.join(sys.argv[2:])) else ()
-
-        print("Building apk...")
-        subprocess.run(['java', '-jar', './tools/apktool_2.9.3.jar', 'b', *debugFlag, 'build/merged', '-o', 'build/apk/tumblr-ykit-unsigned.apk'])
-
-    def alignApk():
-        Task.buildApk()
-
-        print('Aligning apk...')
-        subprocess.run(['tools/zipalign', '-p', '-f', '-v', '4', 'build/apk/tumblr-ykit-unsigned.apk', 'build/apk/tumblr-ykit.apk'])
-
-    def debugKey():
-        if Path('keystores/debug.keystore').exists():
-            return
-
-        print("Debug keystore does not exist, generating...")
-        subprocess.run(['keytool', '-genkey', '-v', '-keystore', 'keystores/debug.keystore', '-alias', 'alias_name', '-keyalg', 'RSA', '-keysize', '2048', '-validity', '10000', '-storepass', '123456', '-keypass', '123456', '-dname', 'CN=, OU=, O=, L=, ST=, C='])
-
-    def signApk():
-        Task.alignApk()
-        Task.debugKey()
-
-        debugFlag = 'd' in ''.join(sys.argv[2:])
-        releaseFlag = 'r' in ''.join(sys.argv[2:])
-
-        if not (debugFlag or releaseFlag):
-            print("Neither debug nor release flag was specified so the apk will not be signed.")
-            return
-
-        if releaseFlag:
-            if not Path('keystores/release.keystore').exists():
-                print("release.keystore is missing from keystores directory")
-                return
-            
-            keystoreOpts = ('--ks', 'keystores/release.keystore',)
-        else:
-            Task.debugKey()
-            keystoreOpts = ('--ks', 'keystores/debug.keystore', '--ks-pass', 'pass:123456',)
-
-        print('Signing apk...')
-        subprocess.run(['java', '-jar', './tools/apksigner.jar', 'sign', *keystoreOpts, 'build/apk/tumblr-ykit.apk'])
-        print("Successfully signed apk in 'build/apk/tumblr-ykit.apk' using " + ('release' if releaseFlag else 'debug') + ' keystore')
-
-    def deployToAndroidStudio():
-        projDir = Path.home() / 'ApkProjects/tumblr-ykit'
-        if not projDir.exists():
-            print("No project named 'tumblr-ykit' in ~/ApkProjects/")
-
-        shutil.copy('build/apk/tumblr-ykit.apk', projDir / 'tumblr-ykit.apk')
-        print("Deployed to Android Studio")
-
-def main():
-
-    match sys.argv[1]:
-        case 'clean':
-            Task.clean()
-        case 'extract':
-            Task.extractApk()
-        case 'assemble':
-            Task.buildApk()
-        case 'build':
-            Task.signApk()
-            
-            if ('a' in ''.join(sys.argv[2:])):
-                Task.deployToAndroidStudio()
-        case 'genclasspath':
-            Task.genClassPath()
-
-if __name__ == '__main__':
-    main()

build.sh

@@ -1 +1 @@
-python build.py "$@"
+python buildtool/build.py "$@"

buildtool/build.py

@@ -0,0 +1,5 @@
+# from task.extract import extract
+from task.patch_resources import patch_resources
+
+if __name__ == '__main__':
+    _ = patch_resources()

buildtool/patch_parser/grammar.lark

@@ -0,0 +1,41 @@
+start: (patch)*
+patch: file_declaration instruction*
+
+file_declaration: "file" string
+
+?string: STRING | RAW_STRING | LONG_STRING
+
+?instruction: insert
+            | preg_replace
+            | define_anchor
+
+insert: "insert" location string
+
+preg_replace: "preg_replace" range PATTERN string
+
+define_anchor: "define_anchor" anchor affix PATTERN
+             | "define_anchor" range_anchor "range" PATTERN
+
+?affix: "before" | "after"
+
+?location: LINE_NUMBER
+         | CHAR_INDEX
+         | LINE_COLUMN
+         | anchor
+
+range: "(" location "-" location ")"
+     | range_anchor
+
+anchor: "@" IDENTIFIER
+range_anchor: "@$" IDENTIFIER
+
+STRING: /"([^"\\]|\\.)*"/
+RAW_STRING: /'[^']*'/
+LONG_STRING: /<<\s*(?P<terminator>[^\n]+)\n.*\n(?P=terminator)/s
+LINE_NUMBER: /ln\d+/
+CHAR_INDEX: /ch\d+/
+LINE_COLUMN: /ln\d+c\d+/
+IDENTIFIER: /[a-zA-Z_][a-zA-Z0-9_]*/
+PATTERN: /\/([^\/\\]|\\.)*\//
+
+%ignore /[\t \f\n]+/

buildtool/patch_parser/polly.py

@@ -0,0 +1,39 @@
+from lark import Lark
+from lark.lexer import Token
+from lark.tree import Branch
+
+import re
+
+def parse_string(token: Token):
+    if token.type == 'STRING':
+        string = token.value[1:-1]
+        string = re.sub(r"\\n","\n", string)
+        string = re.sub(r"\\t","\t", string)
+        string = re.sub(r"\\r","\r", string)
+        string = re.sub(r"\\(.)",r"\1", string)
+        return string
+    elif token.type == 'RAW_STRING':
+        string = token.value[1:-1]
+        return string
+    elif token.type == 'LONG_STRING':
+        string = re.match(re.compile(r"<<\s*(?P<terminator>[^\n]+)\n(.*)\n(?P=terminator)", re.MULTILINE + re.DOTALL),token.value)
+        assert string is not None
+        string = string.group(2)
+        return string
+
+def process_patch(branch: Branch[Token]):
+    # First instruction is always file declaration
+    target_file = parse_string(branch.children[0].children[0]) # pyright: ignore[reportUnknownMemberType, reportArgumentType]
+
+    for inst in branch.children[1:]:
+        match inst.data:
+            case "insert":
+                print(f"Inserting {parse_string(inst.children[1])} at {inst.children[0]} in {target_file}")
+
+if __name__ == '__main__':
+    lark = Lark.open('grammar.lark', rel_to=__file__)
+
+    with open('test.txt', 'r') as f:
+        result = lark.parse(f.read())
+        for patch in result.children:
+            process_patch(patch)

buildtool/task/extract.py

@@ -0,0 +1,25 @@
+from .util import *
+
+import subprocess
+
+def find_apk():
+    apks = list(SOURCE_APK_DIR.glob("*.apk"))
+    if len(apks) == 0:
+        print("Source APK is missing from 'source-apk' directory. Please make sure to copy it to the right directory.")
+        exit(-1)
+    elif len(apks) > 1:
+        print("Multiple APKs were found in the 'source-apk' directory. Please only place the correct APK there.")
+        exit(-1)
+
+    return apks[0]
+
+def extract():
+    target_apk = find_apk()
+
+    # Check if this task is necessary
+    if EXTRACTED_DIR.exists() and EXTRACTED_DIR.stat().st_mtime > target_apk.stat().st_mtime:
+        return "pass"
+    
+    # Extract the apk
+    print("Extracting APK...")
+    _ = subprocess.run(['java', '-jar', './tools/apktool_2.9.3.jar', 'd', str(target_apk.absolute()), '-o', EXTRACTED_DIR])

buildtool/task/fileutil.py

@@ -0,0 +1,24 @@
+from pathlib import Path
+import shutil
+
+def merge_into(src: Path | str, dest: Path | str):
+    src = Path(src)
+    dest = Path(dest)
+    
+    changed = False
+    
+    for (cd, _, files) in src.walk():
+        for file in files:
+            src_file = cd / file
+            dest_file = dest / src_file.relative_to(src)
+            
+            # Don't update if dest is newer than source
+            if dest_file.exists() and src_file.stat().st_mtime < dest_file.stat().st_mtime:
+                continue
+            
+            dest_file.parent.mkdir(parents=True, exist_ok=True)
+            _ = shutil.copyfile(src_file, dest_file)
+            changed = True
+            
+    return changed
+            

buildtool/task/patch_resources.py

@@ -0,0 +1,41 @@
+from .util import *
+from . import fileutil
+
+import shutil
+
+# Task dependencies
+from .extract import extract
+
+def patch_resources():
+    _ = extract()
+    
+    # The way we merge resources is as follows:
+    # First, we copy our raw-resources from src/resources
+    # Second, we apply our patches, which are handled by polly
+    # Finally, we populate all missing entries from build/extracted
+    # The reason we do this is that build/extracted will always have a date greater than src/resources
+    # until the files are modified after first build, which makes simply merging-by-date impossible.
+
+    s = fileutil.merge_into(SRC_RESOURCES_DIR, PATCHED_RSC_DIR)
+    if s:
+        print("Copied custom resources from src/resources")
+    else:
+        print("Skipped copying resources from src/resources")
+        
+    print("Copying original resources...")
+    for (cd, _, files) in EXTRACTED_DIR.walk():
+        for file in files:
+            # Don't copy smali code
+            if file.startswith("smali"):
+                continue
+            
+            src_file = cd / file
+            dest_file = PATCHED_RSC_DIR / src_file.relative_to(EXTRACTED_DIR)
+            
+            # Don't update if dest is newer than source
+            if dest_file.exists() and src_file.stat().st_mtime < dest_file.stat().st_mtime:
+                continue
+            
+            dest_file.parent.mkdir(parents=True, exist_ok=True)
+            _ = shutil.copyfile(src_file, dest_file)
+    

buildtool/task/util.py

@@ -0,0 +1,10 @@
+from pathlib import Path
+
+SOURCE_APK_DIR = Path("source-apk")
+
+BUILD_DIR = Path("build")
+EXTRACTED_DIR = BUILD_DIR / "extracted"
+PATCHED_RSC_DIR = BUILD_DIR / "patched_resources"
+
+SRC_DIR = Path("src")
+SRC_RESOURCES_DIR = SRC_DIR / "resources"

msys.bat

@@ -0,0 +1 @@
+C:\msys64\msys2_shell.cmd -here -defterm -no-start -full-path