From 70618b6b910904af006b41a75d60af8cf78f6b05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=96=9F=E9=85=8C=20=E9=B5=AC=E5=85=84?= Date: Tue, 2 Aug 2022 22:35:37 +0900 Subject: [PATCH] Better caching mechanics --- botanjs/classmap.py | 18 ++++++++++---- botanjs/compressor/closure.py | 9 +++++-- botanjs/dummy.py | 8 +++++- botanjs/service/jclassresv.py | 44 ++++++++++++-------------------- botanjs/service/jwork.py | 4 +-- botanjs/utils.py | 12 +++++++++ tests.py | 47 +++++++++++++++++++++++++++++++++++ 7 files changed, 104 insertions(+), 38 deletions(-) create mode 100644 botanjs/utils.py create mode 100644 tests.py diff --git a/botanjs/classmap.py b/botanjs/classmap.py index bb9b386..2902d7f 100644 --- a/botanjs/classmap.py +++ b/botanjs/classmap.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 -import os; -import re; -import sys; +import os, re, sys from xml.dom import minidom from collections import defaultdict +from botanjs.utils import checksum RegEx_N = re.compile( r""" .* @@ -136,7 +135,7 @@ class ClassMap: return True; - def drawMap( self, ns, ci, ce, cf ): + def drawMap( self, ns, ci, ce, cf, chksum ): nsNode = self.getNode( ns ) # Source every: @@ -150,6 +149,8 @@ class ClassMap: if not srcEvery: nsNode.setAttribute( "src", cf ) + nsNode.setAttribute( "js", chksum["js"] ) + nsNode.setAttribute( "css", chksum["css"] ) for ex in ce: _t = eDef[ ex[0] ] @@ -164,6 +165,8 @@ class ClassMap: cNode.appendChild( impNode ) cNode.setAttribute( "src", cf ) + cNode.setAttribute( "js", chksum["js"] ) + cNode.setAttribute( "css", chksum["css"] ) # the file dose not export classes # Hence it import for itself @@ -187,7 +190,12 @@ class ClassMap: continue ns, ci, ce = classMeta( classFile ) + + chksum = {} + chksum[ "js" ] = checksum( classFile ) + chksum[ "css" ] = checksum( classFile[:-2] + "css" ) + classFile = classFile.replace( self.R + os.path.sep, "" ) - self.drawMap( ns, ci, ce, classFile ) + self.drawMap( ns, ci, ce, classFile, chksum ) return self.DOM.toxml() diff --git a/botanjs/compressor/closure.py b/botanjs/compressor/closure.py index dc943cd..72d08c5 100644 --- a/botanjs/compressor/closure.py +++ b/botanjs/compressor/closure.py @@ -4,11 +4,11 @@ import os from sys import platform from tempfile import NamedTemporaryFile from botanjs.config import Config as config +from botanjs.service.jwork import log COMPILER = config[ "BotanJS" ][ "ClosureCompiler" ] -if not os.path.isfile( COMPILER ): - raise Exception( "Compiler not found" ) +AVAILABLE = os.path.isfile( COMPILER ) COMPILER_OPTIONS = [ @@ -42,6 +42,11 @@ class Wrapper: break def compress( self, loc ): + + if not AVAILABLE: + log.error( "Compiler not found" ) + return + content = "" with open( loc, "rb" ) as f: content = f.read() diff --git a/botanjs/dummy.py b/botanjs/dummy.py index 9c2c9dc..990ebc2 100644 --- a/botanjs/dummy.py +++ b/botanjs/dummy.py @@ -1,7 +1,13 @@ #!/usr/bin/env python3 class log: - def info( self, *args ): + + @staticmethod + def info( *args ): + print( *args ) + + @staticmethod + def error( *args ): print( *args ) class dummyTask( object ): diff --git a/botanjs/service/jclassresv.py b/botanjs/service/jclassresv.py index 0bca229..4f542f5 100644 --- a/botanjs/service/jclassresv.py +++ b/botanjs/service/jclassresv.py @@ -30,7 +30,11 @@ class Resolver: def resource( self, elem ): if "src" in elem.attrib: - return elem.attrib[ "src" ] + return { + "src": elem.attrib[ "src" ] + , "js": elem.attrib[ "js" ] + , "css": elem.attrib[ "css" ] + } parent = self.parentMap[ elem ] @@ -153,9 +157,10 @@ class BotanClassResolver: if src not in classFiles: classFiles.append( src ) - def cssLookup( self, jsList, cssList ): + def cssLookup( self, classList, cssList ): - for f in jsList: + for cdef in classList: + f = cdef[ "src" ] possibleList = [] cssFile = os.path.splitext( f )[0] + ".css" @@ -183,7 +188,7 @@ class BotanClassResolver: if self.CR == None: return None - md5 = hashlib.md5( bytearray( "".join( fileList ), "utf-8" ) ).hexdigest() + md5 = hashlib.md5( bytearray( "|".join( x[mode] for x in fileList ), "utf-8" ) ).hexdigest() cName[0] = oFHash = md5 + "." + mode cFHash = md5 + ".c." + mode @@ -193,19 +198,10 @@ class BotanClassResolver: # Compressed file cFile = os.path.join( self.CR, cFHash ) - dates = list( - os.path.getmtime( os.path.join( self.R, x ) ) - if os.path.exists( os.path.join( self.R, x ) ) else -1 - for x in fileList - ) - - # Root file date - dates.append( os.path.getmtime( os.path.join( self.R, "_this.js" ) ) ); - - if self.flagCompress and self.useCache( cFile, dates ): + if self.flagCompress and self.useCache( cFile ): return self.BotanCache( cFile, cFHash, self.returnDynamic ) - elif self.useCache( oFile, dates ): + elif self.useCache( oFile ): self.JWork.saveCache( oFile # Content is None to initiate a compression @@ -216,17 +212,8 @@ class BotanClassResolver: return self.BotanCache( oFile, oFHash, False ) - def useCache( self, f, dList ): - if not os.path.exists( f ): - return False - - t = os.path.getmtime( f ) - - for i in dList: - if t < i: - return False - - return True + def useCache( self, f ): + return os.path.exists( f ) def compileJs( self, cList, xList ): md5 = [ None ] @@ -244,6 +231,7 @@ class BotanClassResolver: for f in cList: + f = f[ "src" ] path = ( os.path.splitext( f )[0] .replace( PY_SEP, "." ) @@ -285,8 +273,8 @@ class BotanClassResolver: if cacheFile != None: return cacheFile; - # The root file - outputCss = self.BotanFile( "_this.css" ) + struct = "/* @ */" + outputCss = struct + self.BotanFile( "_this.css" ) for f in self.cleanList( cList ): outputCss += self.BotanFile( f ) diff --git a/botanjs/service/jwork.py b/botanjs/service/jwork.py index 2796059..a0aacc8 100644 --- a/botanjs/service/jwork.py +++ b/botanjs/service/jwork.py @@ -1,7 +1,5 @@ #!/usr/bin/env python3 import os -from botanjs.compressor.closure import Wrapper as ClosureWrapper -from botanjs.compressor.yui import Wrapper as YUIWrapper from botanjs.classmap import ClassMap CeleryExists = True @@ -38,6 +36,7 @@ class JWork: @app.task() def compressJs( md5, externs ): + from botanjs.compressor.closure import Wrapper as ClosureWrapper log.info( "Compress js: " + md5 ) w = ClosureWrapper() w.scanExterns( externs ) @@ -45,6 +44,7 @@ class JWork: @app.task() def compressCss( md5 ): + from botanjs.compressor.yui import Wrapper as YUIWrapper log.info( "Compress css: " + md5 ) w = YUIWrapper() w.compress( md5 ) diff --git a/botanjs/utils.py b/botanjs/utils.py new file mode 100644 index 0000000..07bef39 --- /dev/null +++ b/botanjs/utils.py @@ -0,0 +1,12 @@ +from functools import lru_cache +from zlib import adler32 as _HashFunc +HashFunc = lambda v: hex( _HashFunc( v ) )[2:] + + +@lru_cache( maxsize = 1024 ) +def checksum( file_path ): + try: + with open( file_path, "rb" ) as f: + return HashFunc( f.read() ) + except FileNotFoundError: + return HashFunc( b"" ) diff --git a/tests.py b/tests.py new file mode 100644 index 0000000..c7cd4eb --- /dev/null +++ b/tests.py @@ -0,0 +1,47 @@ +#!env/bin/python +import os, sys +sys.path.append( os.path.abspath( "." ) ) + +from botanjs.service.jwork import app, JWork +from botanjs.config import Config as config + +SiteRoot = os.path.abspath( "." ) + +# Setting the SiteRoot for config +config["Paths"]["SiteRoot"] = SiteRoot + +jsCache = config["Paths"]["Cache"] +jsRoot = config["BotanJS"]["SrcDir"] + +bmap = os.path.join( jsCache, "botanjs", "bmap.xml" ) + +app.conf.update( broker_url = config["BotanJS"]["CeleryBroker"] ) + +JWork.buildClassMap.delay( jsRoot, bmap ) + +from botanjs.service.jclassresv import BotanClassResolver as JCResv +srvHandler = JCResv( JWork, jsRoot, bmap, jsCache ) + +import unittest + +class TestStringMethods( unittest.TestCase ): + + # Run each twice to test the cache capabilities + def test_ojscall( self ): + for _ in range(0,2): + s = srvHandler.getAPI( "System", mode = "rjs" ) + self.assertTrue( "BotanJS.define( \"System\" );" in s ) + + def test_import( self ): + for _ in range(0,2): + s = srvHandler.getAPI( "System.Policy", mode = "rjs" ) + self.assertTrue( "BotanJS.define( \"System.Policy\" );" in s ) + self.assertTrue( "BotanJS.define( \"System.Global\" );" in s ) + + def test_cssInheritance( self ): + for _ in range(0,2): + s = srvHandler.getAPI( "System", mode = "rcss" ) + self.assertTrue( "/* @ */" in s ) + +if __name__ == '__main__': + unittest.main()