diff --git a/.gitignore b/.gitignore index 94792fe..f8a3bcc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,8 @@ build/ source-apk/*.apk keystores/*.keystore -tools/* -!tools/smali-2.5.2.jar -!tools/apktool_2.9.3.jar -!tools/apksigner.jar -!tools/d8.jar .idea -*.iml \ No newline at end of file +*.iml + +__pycache__ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e02aef2 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,2 @@ +{ +} \ No newline at end of file diff --git a/build.py b/build.py deleted file mode 100644 index ba0868f..0000000 --- a/build.py +++ /dev/null @@ -1,234 +0,0 @@ -import sys -import shutil -import os -from pathlib import Path -import subprocess - -class BuildError(Exception): - def __init__(self): - super.__init__() - -def getDexName(smaliName): - if smaliName == 'smali': - return 'classes.dex' - elif smaliName.startswith('smali_classes'): - return 'classes' + smaliName[13:] + '.dex' - print("Not a smali name") - raise BuildError() - -class Task: - def clean(): - shutil.rmtree('build') - - def extractApk(): - if Path('build/extracted').exists(): - return - Path('build').mkdir(parents=True, exist_ok=True) - - apk = list(filter(lambda f: f.endswith('.apk'), os.listdir('source-apk'))) - if len(apk) == 0: - print('Please place the APK inside the source-apk directory.') - raise BuildError() - elif len(apk) > 1: - print('More than one apk found in source-apk. Please remove all but one.') - raise BuildError() - - # Extract the apk - print("Extracting APK...") - subprocess.run(['java', '-jar', './tools/apktool_2.9.3.jar', 'd', 'source-apk/' + apk[0], '-o', 'build/extracted']) - - def copySmali(): - Task.extractApk() - - Path('build/smali').mkdir(parents=True, exist_ok=True) - # Copy smali classes that haven't been yet - for file in os.listdir('build/extracted'): - if file != 'smali' and not file.startswith('smali_classes'): - continue - - # Skip if already exists - if Path(f'build/smali/{file}').exists(): - continue - - print(f'Copying {file}...') - shutil.copytree(f'build/extracted/{file}', f'build/smali/{file}') - - # Copy from src - print('Copying ykit smali...') - shutil.copytree('src/smali', 'build/smali', dirs_exist_ok=True) - - def compileSmali(): - Task.copySmali() - - Path('build/dex').mkdir(parents=True, exist_ok=True) - - def shouldCompileSmali(file, dex): - src = f'build/smali/{file}' - ykitsrc = f'src/smali/{file}' - dst = f'build/dex/{dex}' - - # If dest doesn't exist, always build - if not Path(dst).exists(): - return True - - # Check if ykit src dir exists for it, if not then don't recompile - if not Path(ykitsrc).exists(): - return False - - # Get latest file timestamp in ykitsrc - files = list(Path(ykitsrc).rglob('*.smali')) - files = [os.path.getmtime(x) for x in files] - latest = max(files) - - dsttime = os.path.getmtime(dst) - - # If it is newer than the dest, then do recompile - if latest > dsttime: - return True - - # Otherwise, don't - return False - - for file in os.listdir('build/smali'): - dex = getDexName(file) - - if not shouldCompileSmali(file, dex): - print(f"Skipping {file}...") - continue - - print(f"Compiling {file}...") - subprocess.run(['java', '-jar', 'tools/smali-2.5.2.jar', 'a', f'build/smali/{file}', '-o', f'build/dex/{dex}']) - - def genClassPath(): - print("Generating java classpath from dex files...") - for file in os.listdir('build/dex'): - src = f'build/dex/{file}' - dst = f'build/jar/{file[:-4]}.jar' - srctime = os.path.getmtime(src) - dsttime = 0 if not Path(dst).exists() else os.path.getmtime(dst) - - if srctime < dsttime: - print(f"Skipping {file}...") - continue - - print(f"Dex2jar {file}...") - subprocess.run(['java', '-cp', 'tools/d2j/*', 'com.googlecode.dex2jar.tools.Dex2jarCmd', '-f', src, '-o', dst]) - - def compileJava(): - Task.genClassPath() - - sep = ';' if os.name == 'nt' else ':' - javaFiles = [str(x) for x in Path('src/java').rglob('*.java')] - classPath = sep.join([str(x) for x in Path('build/jar').rglob('*.jar')]) - - # Get latest java file timestamp - files = list(Path('src/java').rglob('*.java')) - files = [os.path.getmtime(x) for x in files] - latest = max(files) - - # Get latest dex time or 0 - latest2 = 0 if not Path('build/java_dex/classes.dex').exists() else os.path.getmtime('build/java_dex/classes.dex') - - # If dex file is newer than source, no need to do anything - if latest2 > latest: - print("Skipping java...") - return - - print("Compiling java...") - subprocess.run(['javac', '-cp', classPath, '-d', 'build/java_class', *javaFiles]) - - classFiles = [str(x) for x in Path('build/java_class').rglob('*.class')] - - Path('build/java_dex').mkdir(parents=True, exist_ok=True) - subprocess.run(['java', '-cp', 'tools/d8.jar', 'com.android.tools.r8.D8', *classFiles, '--output', 'build/java_dex']) - - def mergeResources(): - Task.compileSmali() - Task.compileJava() - - # Copy original resources - # on first run only - if not Path('build/merged').exists(): - print('Copying extracted resources...') - shutil.copytree('build/extracted', 'build/merged', dirs_exist_ok=True, ignore=shutil.ignore_patterns('smali', 'smali_classes*')) - - print("Copying ykit resources...") - shutil.copytree('src/resources', 'build/merged', dirs_exist_ok=True) - - print("Copying compiled dex files...") - shutil.copytree('build/dex', 'build/merged', dirs_exist_ok=True) - shutil.copy('build/java_dex/classes.dex', 'build/merged/classes7.dex') # TODO: Unhardcode this - - def buildApk(): - Task.mergeResources() - - debugFlag = ('-d',) if ('d' in ''.join(sys.argv[2:])) else () - - print("Building apk...") - subprocess.run(['java', '-jar', './tools/apktool_2.9.3.jar', 'b', *debugFlag, 'build/merged', '-o', 'build/apk/tumblr-ykit-unsigned.apk']) - - def alignApk(): - Task.buildApk() - - print('Aligning apk...') - subprocess.run(['tools/zipalign', '-p', '-f', '-v', '4', 'build/apk/tumblr-ykit-unsigned.apk', 'build/apk/tumblr-ykit.apk']) - - def debugKey(): - if Path('keystores/debug.keystore').exists(): - return - - print("Debug keystore does not exist, generating...") - subprocess.run(['keytool', '-genkey', '-v', '-keystore', 'keystores/debug.keystore', '-alias', 'alias_name', '-keyalg', 'RSA', '-keysize', '2048', '-validity', '10000', '-storepass', '123456', '-keypass', '123456', '-dname', 'CN=, OU=, O=, L=, ST=, C=']) - - def signApk(): - Task.alignApk() - Task.debugKey() - - debugFlag = 'd' in ''.join(sys.argv[2:]) - releaseFlag = 'r' in ''.join(sys.argv[2:]) - - if not (debugFlag or releaseFlag): - print("Neither debug nor release flag was specified so the apk will not be signed.") - return - - if releaseFlag: - if not Path('keystores/release.keystore').exists(): - print("release.keystore is missing from keystores directory") - return - - keystoreOpts = ('--ks', 'keystores/release.keystore',) - else: - Task.debugKey() - keystoreOpts = ('--ks', 'keystores/debug.keystore', '--ks-pass', 'pass:123456',) - - print('Signing apk...') - subprocess.run(['java', '-jar', './tools/apksigner.jar', 'sign', *keystoreOpts, 'build/apk/tumblr-ykit.apk']) - print("Successfully signed apk in 'build/apk/tumblr-ykit.apk' using " + ('release' if releaseFlag else 'debug') + ' keystore') - - def deployToAndroidStudio(): - projDir = Path.home() / 'ApkProjects/tumblr-ykit' - if not projDir.exists(): - print("No project named 'tumblr-ykit' in ~/ApkProjects/") - - shutil.copy('build/apk/tumblr-ykit.apk', projDir / 'tumblr-ykit.apk') - print("Deployed to Android Studio") - -def main(): - - match sys.argv[1]: - case 'clean': - Task.clean() - case 'extract': - Task.extractApk() - case 'assemble': - Task.buildApk() - case 'build': - Task.signApk() - - if ('a' in ''.join(sys.argv[2:])): - Task.deployToAndroidStudio() - case 'genclasspath': - Task.genClassPath() - -if __name__ == '__main__': - main() \ No newline at end of file diff --git a/build.sh b/build.sh index 7215a8d..2dbbf7b 100644 --- a/build.sh +++ b/build.sh @@ -1 +1 @@ -python build.py "$@" \ No newline at end of file +python buildtool/build.py "$@" \ No newline at end of file diff --git a/buildtool/build.py b/buildtool/build.py new file mode 100644 index 0000000..ccd08b4 --- /dev/null +++ b/buildtool/build.py @@ -0,0 +1,5 @@ +# from task.extract import extract +from task.patch_resources import patch_resources + +if __name__ == '__main__': + _ = patch_resources() \ No newline at end of file diff --git a/buildtool/patch_parser/grammar.lark b/buildtool/patch_parser/grammar.lark new file mode 100644 index 0000000..5b4c49c --- /dev/null +++ b/buildtool/patch_parser/grammar.lark @@ -0,0 +1,41 @@ +start: (patch)* +patch: file_declaration instruction* + +file_declaration: "file" string + +?string: STRING | RAW_STRING | LONG_STRING + +?instruction: insert + | preg_replace + | define_anchor + +insert: "insert" location string + +preg_replace: "preg_replace" range PATTERN string + +define_anchor: "define_anchor" anchor affix PATTERN + | "define_anchor" range_anchor "range" PATTERN + +?affix: "before" | "after" + +?location: LINE_NUMBER + | CHAR_INDEX + | LINE_COLUMN + | anchor + +range: "(" location "-" location ")" + | range_anchor + +anchor: "@" IDENTIFIER +range_anchor: "@$" IDENTIFIER + +STRING: /"([^"\\]|\\.)*"/ +RAW_STRING: /'[^']*'/ +LONG_STRING: /<<\s*(?P[^\n]+)\n.*\n(?P=terminator)/s +LINE_NUMBER: /ln\d+/ +CHAR_INDEX: /ch\d+/ +LINE_COLUMN: /ln\d+c\d+/ +IDENTIFIER: /[a-zA-Z_][a-zA-Z0-9_]*/ +PATTERN: /\/([^\/\\]|\\.)*\// + +%ignore /[\t \f\n]+/ \ No newline at end of file diff --git a/buildtool/patch_parser/polly.py b/buildtool/patch_parser/polly.py new file mode 100644 index 0000000..12e5ab4 --- /dev/null +++ b/buildtool/patch_parser/polly.py @@ -0,0 +1,39 @@ +from lark import Lark +from lark.lexer import Token +from lark.tree import Branch + +import re + +def parse_string(token: Token): + if token.type == 'STRING': + string = token.value[1:-1] + string = re.sub(r"\\n","\n", string) + string = re.sub(r"\\t","\t", string) + string = re.sub(r"\\r","\r", string) + string = re.sub(r"\\(.)",r"\1", string) + return string + elif token.type == 'RAW_STRING': + string = token.value[1:-1] + return string + elif token.type == 'LONG_STRING': + string = re.match(re.compile(r"<<\s*(?P[^\n]+)\n(.*)\n(?P=terminator)", re.MULTILINE + re.DOTALL),token.value) + assert string is not None + string = string.group(2) + return string + +def process_patch(branch: Branch[Token]): + # First instruction is always file declaration + target_file = parse_string(branch.children[0].children[0]) # pyright: ignore[reportUnknownMemberType, reportArgumentType] + + for inst in branch.children[1:]: + match inst.data: + case "insert": + print(f"Inserting {parse_string(inst.children[1])} at {inst.children[0]} in {target_file}") + +if __name__ == '__main__': + lark = Lark.open('grammar.lark', rel_to=__file__) + + with open('test.txt', 'r') as f: + result = lark.parse(f.read()) + for patch in result.children: + process_patch(patch) diff --git a/buildtool/task/extract.py b/buildtool/task/extract.py new file mode 100644 index 0000000..2b6112d --- /dev/null +++ b/buildtool/task/extract.py @@ -0,0 +1,25 @@ +from .util import * + +import subprocess + +def find_apk(): + apks = list(SOURCE_APK_DIR.glob("*.apk")) + if len(apks) == 0: + print("Source APK is missing from 'source-apk' directory. Please make sure to copy it to the right directory.") + exit(-1) + elif len(apks) > 1: + print("Multiple APKs were found in the 'source-apk' directory. Please only place the correct APK there.") + exit(-1) + + return apks[0] + +def extract(): + target_apk = find_apk() + + # Check if this task is necessary + if EXTRACTED_DIR.exists() and EXTRACTED_DIR.stat().st_mtime > target_apk.stat().st_mtime: + return "pass" + + # Extract the apk + print("Extracting APK...") + _ = subprocess.run(['java', '-jar', './tools/apktool_2.9.3.jar', 'd', str(target_apk.absolute()), '-o', EXTRACTED_DIR]) \ No newline at end of file diff --git a/buildtool/task/fileutil.py b/buildtool/task/fileutil.py new file mode 100644 index 0000000..6e6b936 --- /dev/null +++ b/buildtool/task/fileutil.py @@ -0,0 +1,24 @@ +from pathlib import Path +import shutil + +def merge_into(src: Path | str, dest: Path | str): + src = Path(src) + dest = Path(dest) + + changed = False + + for (cd, _, files) in src.walk(): + for file in files: + src_file = cd / file + dest_file = dest / src_file.relative_to(src) + + # Don't update if dest is newer than source + if dest_file.exists() and src_file.stat().st_mtime < dest_file.stat().st_mtime: + continue + + dest_file.parent.mkdir(parents=True, exist_ok=True) + _ = shutil.copyfile(src_file, dest_file) + changed = True + + return changed + \ No newline at end of file diff --git a/buildtool/task/patch_resources.py b/buildtool/task/patch_resources.py new file mode 100644 index 0000000..d1eb2fd --- /dev/null +++ b/buildtool/task/patch_resources.py @@ -0,0 +1,41 @@ +from .util import * +from . import fileutil + +import shutil + +# Task dependencies +from .extract import extract + +def patch_resources(): + _ = extract() + + # The way we merge resources is as follows: + # First, we copy our raw-resources from src/resources + # Second, we apply our patches, which are handled by polly + # Finally, we populate all missing entries from build/extracted + # The reason we do this is that build/extracted will always have a date greater than src/resources + # until the files are modified after first build, which makes simply merging-by-date impossible. + + s = fileutil.merge_into(SRC_RESOURCES_DIR, PATCHED_RSC_DIR) + if s: + print("Copied custom resources from src/resources") + else: + print("Skipped copying resources from src/resources") + + print("Copying original resources...") + for (cd, _, files) in EXTRACTED_DIR.walk(): + for file in files: + # Don't copy smali code + if file.startswith("smali"): + continue + + src_file = cd / file + dest_file = PATCHED_RSC_DIR / src_file.relative_to(EXTRACTED_DIR) + + # Don't update if dest is newer than source + if dest_file.exists() and src_file.stat().st_mtime < dest_file.stat().st_mtime: + continue + + dest_file.parent.mkdir(parents=True, exist_ok=True) + _ = shutil.copyfile(src_file, dest_file) + \ No newline at end of file diff --git a/buildtool/task/util.py b/buildtool/task/util.py new file mode 100644 index 0000000..cc4701e --- /dev/null +++ b/buildtool/task/util.py @@ -0,0 +1,10 @@ +from pathlib import Path + +SOURCE_APK_DIR = Path("source-apk") + +BUILD_DIR = Path("build") +EXTRACTED_DIR = BUILD_DIR / "extracted" +PATCHED_RSC_DIR = BUILD_DIR / "patched_resources" + +SRC_DIR = Path("src") +SRC_RESOURCES_DIR = SRC_DIR / "resources" \ No newline at end of file diff --git a/msys.bat b/msys.bat new file mode 100644 index 0000000..346a638 --- /dev/null +++ b/msys.bat @@ -0,0 +1 @@ +C:\msys64\msys2_shell.cmd -here -defterm -no-start -full-path \ No newline at end of file diff --git a/tools/apksigner.jar b/tools/apksigner.jar deleted file mode 100644 index f4adcff..0000000 Binary files a/tools/apksigner.jar and /dev/null differ diff --git a/tools/d8.jar b/tools/d8.jar deleted file mode 100644 index a93636e..0000000 Binary files a/tools/d8.jar and /dev/null differ