From 01dca50e2ee620a7c7148fbacfb1ff9df63b75df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=96=9F=E9=85=8C=20=E9=B5=AC=E5=85=84?= Date: Sun, 5 Sep 2021 23:16:37 +0800 Subject: [PATCH] Added docker windows container for dev --- .dockerignore | 13 +++++++++ README.md | 26 +++++------------ botan-rebuild.py | 2 +- botanjs/compressor/closure.py | 5 ++-- botanjs/compressor/yui.py | 6 +++- botanjs/dummy.py | 2 +- botanjs/service/jclassresv.py | 43 ++++++++++++++++++++-------- botanjs/service/jwork.py | 5 ++-- botanjs/service/webapi.py | 11 +++++-- main.py | 2 +- settings.ini | 3 +- windows/app/Dockerfile | 12 ++++++++ windows/docker-compose.yml | 41 ++++++++++++++++++++++++++ windows/jre/Dockerfile | 9 ++++++ windows/pyrt/dockerfile | 54 +++++++++++++++++++++++++++++++++++ windows/redis/Dockerfile | 27 ++++++++++++++++++ 16 files changed, 217 insertions(+), 44 deletions(-) create mode 100644 .dockerignore create mode 100644 windows/app/Dockerfile create mode 100644 windows/docker-compose.yml create mode 100644 windows/jre/Dockerfile create mode 100644 windows/pyrt/dockerfile create mode 100644 windows/redis/Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..2722717 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.dockerignore +.git +.gitignore +*~ +*.swp +*.pyc +Dockerfile +k8s.yaml +windows/* +__pycache__ +cache/* +logs/* +env/* diff --git a/README.md b/README.md index 29c63d2..ca8d8ea 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ # BotanJS A working concept of Js/Css framework for web browsers -### Showcase -- Pending +### Disclaimer +- This is a working concept. So it works on me. And may have a bunch of useless dependency. Use at your own risks! + +### Try it - Docker for windows (using windows container with nanoserver) +- docker -f windows/docker-compose.yml build +- docker -f windows/docker-compose.yml up ### Features - Compressable by Closure Compiler (Advanced Compression) @@ -10,29 +14,13 @@ A working concept of Js/Css framework for web browsers - Everything is merged into one file for that page - css class inheritance -### Disclaimer -- This is a working concept. So it works on me. And may have a bunch of useless dependency. Use at your own risks! - ### Documentation - Will be added later ### Prerequisties - python3 -- virtualenv - -#### For Service.WebAPI +- virtualenv ( optional ) - pip install Flask - pip install Celery - pip install redis - pip install compressinja - -### Before start, run -``` -virtualenv env -./botan-rebuild -``` - -#### To start just run ( in virtualenv ) -``` -./botan-start -``` diff --git a/botan-rebuild.py b/botan-rebuild.py index 4ea03c8..421f4a8 100755 --- a/botan-rebuild.py +++ b/botan-rebuild.py @@ -12,6 +12,6 @@ config["Paths"]["SiteRoot"] = SiteRoot; bmap = os.path.join( config["Paths"]["Cache"], "botanjs", "bmap.xml" ) -app.conf.update( BROKER_URL = config["BotanJS"]["CeleryBroker"] ) +app.conf.update( broker_url = config["BotanJS"]["CeleryBroker"] ) JWork.buildClassMap.delay( config["BotanJS"]["SrcDir"], bmap ) diff --git a/botanjs/compressor/closure.py b/botanjs/compressor/closure.py index 9a9c42e..dc943cd 100644 --- a/botanjs/compressor/closure.py +++ b/botanjs/compressor/closure.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +from sys import platform from tempfile import NamedTemporaryFile from botanjs.config import Config as config @@ -45,8 +46,6 @@ class Wrapper: with open( loc, "rb" ) as f: content = f.read() - with NamedTemporaryFile() as f: + with NamedTemporaryFile( delete = ( not platform == "win32" ) ) as f: f.write( content[12:-5] ) os.system( self.C + self.E + " --js " + f.name + " --js_output_file " + loc[:-3] + ".c.js" ) - - diff --git a/botanjs/compressor/yui.py b/botanjs/compressor/yui.py index b4dd0ff..8a665f1 100644 --- a/botanjs/compressor/yui.py +++ b/botanjs/compressor/yui.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import os +from sys import platform from botanjs.config import Config as config COMPILER = config[ "BotanJS" ][ "YuiCompressor" ] @@ -20,5 +21,8 @@ class Wrapper: self.C = "java -jar " + COMPILER + " " + " ".join( COMPILER_OPTIONS ) def compress( self, loc ): - os.system( self.C + " " + loc + " -o " + loc[:-4] + ".c.css" ) + if platform == "win32": + loc = loc.replace( "C:", "" ).replace( "\\\\", "/" ) + + os.system( self.C + " " + loc + " -o " + loc[:-4] + ".c.css" ) diff --git a/botanjs/dummy.py b/botanjs/dummy.py index ce93373..9c2c9dc 100644 --- a/botanjs/dummy.py +++ b/botanjs/dummy.py @@ -18,7 +18,7 @@ class dummyTask( object ): pass class dummyConf: - def update( self, BROKER_URL = None ): + def update( self, broker_url = None ): pass class app: diff --git a/botanjs/service/jclassresv.py b/botanjs/service/jclassresv.py index 536c2d6..6989a41 100644 --- a/botanjs/service/jclassresv.py +++ b/botanjs/service/jclassresv.py @@ -99,7 +99,9 @@ class BotanClassResolver: classMap = "" flagCompress = True + oModeIfSizelt = 50 * 1024 returnHash = False + returnDynamic = False resv = None def __init__( self, jwork, BotanRoot, classMap, cacheRoot ): @@ -122,12 +124,17 @@ class BotanClassResolver: return content - def BotanCache( self, t ): - content = "" - with open( t, "r" ) as f: - content = f.read() + def BotanCache( self, srcFile, fileHash, hashContentDynamic ): - return content + for _ in [0]: + if hashContentDynamic and os.path.getsize( srcFile ) < self.oModeIfSizelt: + break + + if self.returnHash: + return fileHash + + with open( srcFile, "r" ) as f: + return f.read() def cleanList( self, lista ): olist = [] @@ -196,7 +203,7 @@ class BotanClassResolver: dates.append( os.path.getmtime( os.path.join( self.R, "_this.js" ) ) ); if self.flagCompress and self.useCache( cFile, dates ): - return cFHash if self.returnHash else self.BotanCache( cFile ) + return self.BotanCache( cFile, cFHash, self.returnDynamic ) elif self.useCache( oFile, dates ): self.JWork.saveCache( @@ -207,7 +214,7 @@ class BotanClassResolver: , os.path.join( self.R, "externs" ) ) - return oFHash if self.returnHash else self.BotanCache( oFile ) + return self.BotanCache( oFile, oFHash, False ) def useCache( self, f, dList ): if not os.path.exists( f ): @@ -248,7 +255,7 @@ class BotanClassResolver: outputJs = wrapScope( outputJs ) - [ self.JWork.saveCache if self.returnHash else self.JWork.saveCache ][0] ( + self.JWork.saveCache( os.path.join( self.CR, md5[0] ) , outputJs , "js" @@ -283,9 +290,7 @@ class BotanClassResolver: for f in self.cleanList( cList ): outputCss += self.BotanFile( f ) - [ self.JWork.saveCache if self.returnHash else self.JWork.saveCache ][0] ( - os.path.join( self.CR, md5[0] ), outputCss, "css" - ) + self.JWork.saveCache( os.path.join( self.CR, md5[0] ), outputCss, "css" ) if self.returnHash: return md5[0] @@ -299,13 +304,26 @@ class BotanClassResolver: flag = mode[0] requestAPIs = code + # Return compressed contents if possible + # otherwise return raw contents if flag == "o": mode = mode[1:] + + # Return raw contents only elif flag == "r": mode = mode[1:] self.flagCompress = False + + # Return hashed filenames only + elif flag == "h": + mode = mode[1:] + self.returnHash = True + + # Return hashed filenames if content is larger than self.oModeIfSizelt bytes + # otherwise act as "o" mode else: self.returnHash = True + self.returnDynamic = True try: requestAPIs = ( @@ -329,7 +347,8 @@ class BotanClassResolver: for apis in requestAPIs: - if apis == None: continue + if not apis: + continue classList = [] lookupList = imports diff --git a/botanjs/service/jwork.py b/botanjs/service/jwork.py index 1a3da0f..2796059 100644 --- a/botanjs/service/jwork.py +++ b/botanjs/service/jwork.py @@ -17,7 +17,7 @@ if CeleryExists: if os.path.exists( "settings.ini" ): from botanjs.config import Config - app.conf.update( BROKER_URL = Config["BotanJS"]["CeleryBroker"] ) + app.conf.update( broker_url = Config["BotanJS"]["CeleryBroker"] ) else: from botanjs.dummy import app @@ -34,7 +34,7 @@ class JWork: if mode == "js": JWork.compressJs.delay( location, externs ) elif mode == "css": - JWork.compressCss.delay( location ) + JWork.compressCss( location ) @app.task() def compressJs( md5, externs ): @@ -54,5 +54,6 @@ class JWork: log.info( "Building Class Map" ) c = ClassMap( src ) + os.makedirs( os.path.dirname( location ), exist_ok = True ) with open( location, "w" ) as f: f.write( c.build() ) diff --git a/botanjs/service/webapi.py b/botanjs/service/webapi.py index ab2dd5c..c28ec01 100644 --- a/botanjs/service/webapi.py +++ b/botanjs/service/webapi.py @@ -23,9 +23,9 @@ class WebAPI: self.BMap = os.path.join( self.BCache, "bmap.xml" ) if brokerURL != None: - CeleryApp.conf.update( BROKER_URL = brokerURL ) + CeleryApp.conf.update( broker_url = brokerURL ) - self.app = Flask( __name__, static_url_path = self.BCache, static_folder = self.BCache ) + self.app = Flask( __name__, static_url_path = "/cache/botanjs", static_folder = self.BCache ) self.app.jinja_env.add_extension( "compressinja.html.HtmlCompressor" ) self.app.add_url_rule( "/" , view_func = self.index ) @@ -33,6 +33,7 @@ class WebAPI: self.app.add_url_rule( "//" , view_func = self.api_request ) def run( self, *args, **kwargs ): + JWork.buildClassMap( self.BRoot, self.BMap ) return self.app.run( *args, **kwargs ) def index( self ): @@ -40,6 +41,10 @@ class WebAPI: def api_request( self, mode, code ): + if mode == "rebuild": + JWork.buildClassMap.delay( self.BRoot, self.BMap ) + return "OK", 200 + if code == "zpayload": code = request.args.get( "p" ) @@ -53,6 +58,8 @@ class WebAPI: srvHandler = JCResv( JWork, self.BRoot, self.BMap, self.BCache ) return Response( srvHandler.getAPI( code, mode = mode ), mimetype = t ) except Exception as e: + if self.app.config[ "DEBUG" ]: + raise return str(e), 404 diff --git a/main.py b/main.py index d3ade1d..9fdfdda 100644 --- a/main.py +++ b/main.py @@ -8,7 +8,7 @@ from botanjs.service.webapi import WebAPI SiteRoot = os.path.abspath( "." ) # Setting the SiteRoot for config -config["Paths"]["SiteRoot"] = SiteRoot; +config["Paths"]["SiteRoot"] = SiteRoot service = WebAPI( jsCache = config["Paths"]["Cache"] diff --git a/settings.ini b/settings.ini index 358d491..790c081 100644 --- a/settings.ini +++ b/settings.ini @@ -12,9 +12,8 @@ Cache = ${SiteRoot}/cache [BotanJS] SrcDir = ${Paths:Runtime}/botanjs/src -CeleryBroker = redis://:${REDIS_PASS}@123.123.123.123:1234/9 -REDIS_PASS = PASSWORD_FOR_REDIS_DB +CeleryBroker = redis://:@redis:6379/9 ClosureCompiler = /opt/utils/closure.jar YuiCompressor = /opt/utils/yuicompressor.jar diff --git a/windows/app/Dockerfile b/windows/app/Dockerfile new file mode 100644 index 0000000..f6e3b5d --- /dev/null +++ b/windows/app/Dockerfile @@ -0,0 +1,12 @@ +FROM astrojs/jre-nanoserver-1809:latest + +RUN pip3 install Flask redis compressinja celery + +RUN New-Item -ItemType "Directory" -Path /opt/utils; \ + New-Item -ItemType "Directory" -Path /app; \ + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; \ + Invoke-WebRequest -UseBasicParsing -Uri 'https://github.com/tgckpg/BotanJS/releases/download/compressors/closure.jar' -OutFile '/opt/utils/closure.jar' ; \ + Invoke-WebRequest -UseBasicParsing -Uri 'https://github.com/tgckpg/BotanJS/releases/download/compressors/yuicompressor.jar' -OutFile '/opt/utils/yuicompressor.jar' ; + +EXPOSE 5000 +WORKDIR /app diff --git a/windows/docker-compose.yml b/windows/docker-compose.yml new file mode 100644 index 0000000..385eb5e --- /dev/null +++ b/windows/docker-compose.yml @@ -0,0 +1,41 @@ +version: '3.9' + +services: + python: + build: pyrt + image: astrojs/pyrt-nanoserver-1809 + jre: + build: jre + image: astrojs/jre-nanoserver-1809 + redis: + container_name: astrojsdev_redis + build: redis + web: + container_name: astrojsdev_app + image: astrojs/app + hostname: astrojs.default + build: + context: ../ + dockerfile: windows/app/Dockerfile + command: [ "python", "main.py" ] # [ "ping", "127.0.0.1", "-n", "9999" ] + depends_on: + - redis + ports: + - 5000:5000 + volumes: + - ../:C:/app + - cache:C:/app/cache + tasks: + container_name: astrojsdev_compiler + image: astrojs/app + command: [ "celery", "-A", "botanjs.service.jwork", "worker", "-l", "info", "--pool=solo" ] + depends_on: + - redis + - web + volumes: + - ../:C:/app + - cache:C:/app/cache + +volumes: + cache: + temp: diff --git a/windows/jre/Dockerfile b/windows/jre/Dockerfile new file mode 100644 index 0000000..1844c71 --- /dev/null +++ b/windows/jre/Dockerfile @@ -0,0 +1,9 @@ +FROM astrojs/pyrt-nanoserver-1809:latest as base + +ENV JAVA_HOME=C:\\openjdk-11 +ENV JAVA_VERSION=11.0.12 + +RUN setx PATH "$Env:JAVA_HOME\bin`;$Env:Path" /M + +COPY --from=openjdk:11-jre-nanoserver-1809 /openjdk-11 /openjdk-11 +RUN echo Verifying install ... && echo java --version && java --version && echo Complete. diff --git a/windows/pyrt/dockerfile b/windows/pyrt/dockerfile new file mode 100644 index 0000000..ad8b14e --- /dev/null +++ b/windows/pyrt/dockerfile @@ -0,0 +1,54 @@ +FROM mcr.microsoft.com/powershell:nanoserver-1809 + +SHELL [ "pwsh", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';" ] + +RUN $url = 'https://www.python.org/ftp/python/3.7.6/python-3.7.6-embed-amd64.zip'; \ + Write-host "downloading: $url"; \ + [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; \ + New-Item -ItemType Directory /installer > $null ; \ + Invoke-WebRequest -Uri $url -outfile /installer/Python.zip -verbose; \ + Expand-Archive /installer/Python.zip -DestinationPath /Python; \ + Move-Item /Python/python37._pth /Python/python37._pth.save + +### Begin workaround ### +# Note that changing user on nanoserver is not recommended +# See, https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-containers/container-base-images#base-image-differences +# But we are working around a bug introduced in the nanoserver image introduced in 1809 +USER ContainerAdministrator + +# This is basically the correct code except for the /M +RUN setx PATH "$Env:Path`C:\Python`;C:\Python\Scripts`;" /M + +# We can't +# USER ContainerUser +### End workaround ### + +# if this is called "PIP_VERSION", pip explodes with "ValueError: invalid truth value ''" +ENV PYTHON_PIP_VERSION 21.2.4 +# https://github.com/pypa/get-pip +ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/4b85d3add912c861aea4a9feaae737a5b7b9cb1c/public/get-pip.py +ENV PYTHON_GET_PIP_SHA256 ced8c71489cd46c511677bfe423f37eb88f08f29e9af36ef2679091ec7122d4f + +RUN Write-Host ('Downloading get-pip.py ({0}) ...' -f $env:PYTHON_GET_PIP_URL); \ + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; \ + Invoke-WebRequest -Uri $env:PYTHON_GET_PIP_URL -OutFile 'get-pip.py'; \ + Write-Host ('Verifying sha256 ({0}) ...' -f $env:PYTHON_GET_PIP_SHA256); \ + if ((Get-FileHash 'get-pip.py' -Algorithm sha256).Hash -ne $env:PYTHON_GET_PIP_SHA256) { \ + Write-Host 'FAILED!'; \ + exit 1; \ + }; \ + \ + Write-Host ('Installing pip=={0} ...' -f $env:PYTHON_PIP_VERSION); \ + python get-pip.py \ + --disable-pip-version-check \ + --no-cache-dir \ + ('pip=={0}' -f $env:PYTHON_PIP_VERSION) \ + ; \ + Remove-Item get-pip.py -Force; \ + \ + Write-Host 'Verifying pip install ...'; \ + pip --version; \ + \ + Write-Host 'Complete.' + +CMD [ "python.exe" ] diff --git a/windows/redis/Dockerfile b/windows/redis/Dockerfile new file mode 100644 index 0000000..65e3271 --- /dev/null +++ b/windows/redis/Dockerfile @@ -0,0 +1,27 @@ +FROM mcr.microsoft.com/powershell:nanoserver-1809 + +SHELL [ "pwsh", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';" ] + +RUN [Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12; \ + Invoke-WebRequest -UseBasicParsing -Uri 'https://github.com/tporadowski/redis/releases/download/v5.0.10/Redis-x64-5.0.10.zip' -OutFile 'Redis-x64-5.0.10.zip' ; \ + Expand-Archive Redis-x64-5.0.10.zip -dest 'C:\\Program Files\\Redis\\' ; \ + Remove-Item Redis-x64-5.0.10.zip -Force + +User ContainerAdministrator +RUN setx Path "C:\Program` Files\Redis`;$Env:Path" /M; +User ContainerUser + +WORKDIR 'C:\\Program Files\\Redis' + +RUN Get-Content redis.windows.conf | Where { $_ -notmatch 'bind 127.0.0.1' } | Set-Content redis.openport.conf ; \ + Get-Content redis.openport.conf | Where { $_ -notmatch 'protected-mode yes' } | Set-Content redis.unprotected.conf ; \ + Add-Content redis.unprotected.conf 'protected-mode no' ; \ + Add-Content redis.unprotected.conf 'bind 0.0.0.0' ; \ + Get-Content redis.unprotected.conf + +EXPOSE 6379 + +# Define our command to be run when launching the container +CMD .\\redis-server.exe .\\redis.unprotected.conf --port 6379 ; \ + Write-Host Redis Started... ; \ + while ($true) { Start-Sleep -Seconds 3600 }