Deprecating old approach

botanjs/old/classmap.py

@@ -0,0 +1,204 @@
+import os, re, sys
+
+from xml.dom import minidom
+from collections import defaultdict
+
+from botanjs.config import DEBUG
+
+if DEBUG:
+	from botanjs.utils import checksum_r as checksum
+else:
+	from botanjs.utils import checksum
+
+RegEx_N = re.compile( r"""
+		.*
+		__namespace
+		\s*\(
+			\s*(['"])([^\1]+)\1
+		\s*\)
+		.*
+		""", re.X )
+
+RegEx_I = re.compile( r"""
+		.*
+		__import
+		\s*\(
+			\s*(['"])([^\1]+)\1
+		\s*\)
+		.*
+		""", re.X )
+
+RegEx_V = re.compile( r"""
+		.*
+		ns
+		\s*\[
+		\s*NS_INVOKE
+		\s*\]
+		\s*\(
+			\s*(['"])([^\1]+)\1
+		\s*\)
+		.*
+		""", re.X )
+
+RegEx_E = re.compile( r"""
+		.*
+		ns
+		\s*\[
+		\s*NS_EXPORT
+		\s*\]
+		\s*\(
+			\s*EX_([A-Z_]+[A-Z])
+			\s*,
+			\s*(['"])([^\1]+)\2
+			\s*,
+			[^\)]+
+		\s*\)
+		.*
+		""", re.X )
+
+def classMeta( cf ):
+	ns = ""
+	imps = list()
+	exps = list()
+	for line in open( cf, "r" ):
+
+		m = RegEx_N.match( line )
+		if m:
+			ns = m.group(2)
+			continue
+
+		m = RegEx_I.match( line )
+		if m:
+			imps.append( m.group(2) )
+			continue
+
+		m = RegEx_V.match( line )
+		if m:
+			imps.append( ns + "." + m.group(2) )
+			continue
+
+		m = RegEx_E.match( line )
+		if m:
+			exps.append( [ m.group(1), m.group(3) ] )
+			continue
+
+	return [ ns, imps, exps ]
+
+def className( classFile ):
+	return ( os.path
+				.splitext( classFile )[0]
+				.replace( os.sep, "." )
+				.replace( "._this", "" )
+				.replace( "..BotanJS.", "" ) )
+
+# __export types definition => nodeName
+EX_CLASS = "class"
+EX_FUNC = "method"
+eDef = defaultdict( lambda: 'prop', { "CLASS": EX_CLASS, "FUNC": EX_FUNC } )
+
+class ClassMap:
+	head = None
+	DOM = None
+	R = None
+
+	def __init__( self, BotanRoot ):
+		self.R = BotanRoot
+		self.DOM = minidom.parseString( "<BotanJS></BotanJS>" )
+		head = os.path.join( self.R, "_this.js" )
+
+	def getNode( self, name, t = EX_CLASS ):
+		paths = name.split( "." )
+
+		currentNode = self.DOM.firstChild
+
+		# Step down the path and create the path if necessary
+		for path in paths:
+
+			l = currentNode.childNodes
+			for i in l:
+				if i.getAttribute( "name" ) == path:
+					currentNode = i
+					break
+
+			if currentNode.getAttribute( "name" ) != path:
+				newNode = self.DOM.createElement( t )
+				newNode.setAttribute( "name", path )
+				currentNode.appendChild( newNode )
+				currentNode = newNode
+
+		return currentNode
+
+
+
+	def skipFile( self, cf ):
+
+		if os.path.splitext( cf )[1] == ".js":
+			if cf == self.head:
+				return True
+			return False
+
+		return True;
+
+	def drawMap( self, ns, ci, ce, cf, chksum ):
+		nsNode = self.getNode( ns )
+
+		# Source every:
+		# Since namespace may differ from file name
+		# ns.__export may export to a different level
+		# Source the exported val explicitly means
+		# exports only available when source file is imported
+		srcEvery = ( ns != className( cf ) )
+
+		cf = cf.replace( self.R, "" )
+
+		if not srcEvery:
+			nsNode.setAttribute( "src", cf )
+			nsNode.setAttribute( "js", chksum["js"] )
+			nsNode.setAttribute( "css", chksum["css"] )
+
+		for ex in ce:
+			_t = eDef[ ex[0] ]
+			cNode = self.getNode( ns + "." + ex[1], t = _t )
+
+			if srcEvery:
+				# The import is for the defined class
+				if _t == EX_CLASS:
+					for imp in ci:
+						impNode = self.DOM.createElement( "import" )
+						impNode.appendChild( self.DOM.createTextNode( imp ) )
+						cNode.appendChild( impNode )
+
+				cNode.setAttribute( "src", cf )
+				cNode.setAttribute( "js", chksum["js"] )
+				cNode.setAttribute( "css", chksum["css"] )
+
+		# the file dose not export classes
+		# Hence it import for itself
+		if not srcEvery:
+			for imp in ci:
+				impNode = self.DOM.createElement( "import" )
+				impNode.appendChild( self.DOM.createTextNode( imp ) )
+				nsNode.appendChild( impNode )
+
+
+	def build( self ):
+		for root, dirs, files in os.walk( self.R ):
+
+			if root == self.R:
+				dirs.remove("externs")
+
+			for name in files:
+				classFile = os.path.join( root, name )
+
+				if self.skipFile( classFile ):
+					continue
+
+				ns, ci, ce = classMeta( classFile )
+
+				chksum = {}
+				chksum[ "js" ] = checksum( classFile )
+				chksum[ "css" ] = checksum( classFile[:-2] + "css" )
+
+				classFile = classFile.replace( self.R + os.path.sep, "" )
+				self.drawMap( ns, ci, ce, classFile, chksum )
+		return self.DOM.toxml()

botanjs/old/compressor/closure.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+import os
+from sys import platform
+from tempfile import NamedTemporaryFile
+from botanjs.config import Config as config
+from botanjs.service.jwork import log
+
+COMPILER = config[ "BotanJS" ][ "ClosureCompiler" ]
+
+AVAILABLE = os.path.isfile( COMPILER )
+
+
+COMPILER_OPTIONS = [
+	"--compilation_level ADVANCED_OPTIMIZATIONS"
+	, "--output_wrapper=\"(function(){%output%})();\""
+]
+
+
+class Wrapper:
+
+	C = None
+	# externs
+	E = ""
+
+	def __init__( self ):
+		self.C = "java -jar -Xmx64M "+ COMPILER + " " + " ".join( COMPILER_OPTIONS )
+
+	def scanExterns( self, sdir ):
+		for root, dirs, files in os.walk( sdir ):
+			# Split file extensions
+			files = list( os.path.splitext( x ) for x in files )
+			files.sort()
+			for f in files:
+				files.remove( f ) if f[1] != ".js" else None
+
+			self.E = " --externs " + " --externs ".join(
+				os.path.join( root, x )
+				# join back extensions
+				for x in list( "".join( x ) for x in files )
+			)
+			break
+
+	def compress( self, loc ):
+
+		if not AVAILABLE:
+			log.error( "Compiler not found" )
+			return
+
+		content = ""
+		with open( loc, "rb" ) as f:
+			content = f.read()
+
+		with NamedTemporaryFile( delete = ( not platform == "win32" ) ) as f:
+			f.write( content[12:-5] )
+			os.system( self.C + self.E + " --js " + f.name + " --js_output_file " + loc[:-3] + ".c.js" )

botanjs/old/compressor/yui.py

@@ -0,0 +1,28 @@
+#!/usr/bin/env python3
+
+import os
+from sys import platform
+from botanjs.config import Config as config
+
+COMPILER = config[ "BotanJS" ][ "YuiCompressor" ]
+
+if not os.path.isfile( COMPILER ):
+	raise Exception( "Compiler not found" )
+
+COMPILER_OPTIONS = [
+	"--type css"
+]
+
+class Wrapper:
+
+	C = None
+
+	def __init__( self ):
+		self.C = "java -jar " + COMPILER + " " + " ".join( COMPILER_OPTIONS )
+
+	def compress( self, loc ):
+
+		if platform == "win32":
+			loc = loc.replace( "C:", "" ).replace( "\\\\", "/" )
+
+		os.system( self.C + " " + loc + " -o " + loc[:-4] + ".c.css" )

botanjs/old/config.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+import configparser, os
+
+Config = configparser.ConfigParser( interpolation = configparser.ExtendedInterpolation() )
+Config.read( "settings.ini" )
+
+DEBUG = os.getenv( "DEBUG" )
+if DEBUG is None:
+	DEBUG = Config.getboolean( "Env", "Debug" )
+else:
+	Config[ "Env" ][ "Debug" ] = str( DEBUG == "1" )
+
+REDIS_CONN = os.getenv( "REDIS_CONN" )
+if not REDIS_CONN is None:
+	Config[ "Redis" ][ "ConnStr" ] = REDIS_CONN

botanjs/old/dummy.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+class log:
+
+	@staticmethod
+	def info( *args ):
+		print( *args )
+
+	@staticmethod
+	def error( *args ):
+		print( *args )
+
+class dummyTask( object ):
+
+	Func = None
+
+	def delay( self, *args ):
+		self.Func( *args )
+
+	def __init__( self, *args ):
+		self.Func = args[0]
+
+	def __call__( self, *args ):
+		pass
+
+class dummyConf:
+	def update( self, broker_url = None ):
+		pass
+
+class app:
+	conf = dummyConf()
+
+	def task():
+		return dummyTask

botanjs/old/service/jclassresv.py

@@ -0,0 +1,368 @@
+import os, re, base64, zlib, hashlib, binascii
+import xml.etree.ElementTree as ET
+
+PY_SEP = os.path.sep
+
+def wrapScope( src ):
+	return "(function(){" + src + "})();"
+
+class Resolver:
+	resolved = None
+
+	bMap = None
+	parentMap = None
+
+	EX_PROP = "prop"
+	EX_FUNC = "method"
+	EX_CLASS = "class"
+
+	_rLookup = None
+
+	def __init__( self, classMap ):
+		self.classMap = classMap
+		self._rLookup = {}
+		self._reload()
+
+	def _reload( self ):
+		self.bMap = ET.parse( self.classMap )
+		# ElementTree does not support Element.find( ".." )
+		# Make a parent map to work around
+		self.parentMap = { c:p for p in self.bMap.iter() for c in p }
+
+		self.resolved = []
+
+	def resource( self, elem ):
+		if "src" in elem.attrib:
+			key = elem.attrib[ "src" ]
+			if not key in self._rLookup:
+				self._rLookup[ key ] = {
+					"src": elem.attrib[ "src" ]
+					, "js": elem.attrib[ "js" ]
+					, "css": elem.attrib[ "css" ]
+				}
+			return self._rLookup[ key ]
+
+		parent = self.parentMap[ elem ]
+
+		if parent is not None:
+			return self.resource( parent )
+
+	def locate( self, key ):
+		return self._rLookup.get( key )
+
+	def resolve( self, c, classList ):
+		self.resolved = []
+		self.__resolve( c, classList )
+
+	def __resolve( self, c, classList ):
+
+		lista = list( "[@name=\"" + x + "\"]" for x in c.split(".") )
+
+		fx = "/".join( self.EX_CLASS + x for x in lista[:-1] )
+
+		# resolve wildcard A.B.*
+		if  c[-2:] == ".*":
+			elem = self.bMap.findall( ".//" + fx + self.EX_CLASS )
+
+			if elem == None:
+				raise LookupError( "Namespace does not exists or contains no classes: " + c )
+
+			c = c[0:-1]
+			for cl in elem:
+				cl = c + cl.attrib[ "name" ]
+
+				if cl not in self.resolved:
+					self.__resolve( cl, classList )
+
+			return
+
+		it = lista[-1]
+		# Test if class
+		elem = self.bMap.find( ".//" + fx + self.EX_CLASS + it )
+
+		if elem == None:
+			if fx != '': fx += "/"
+
+			# Test if prop
+			elem = self.bMap.find( ".//" + fx + self.EX_PROP + it )
+
+			# test if func
+			if elem == None:
+				elem = self.bMap.find( ".//" + fx + self.EX_FUNC + it )
+
+			if elem == None:
+				raise LookupError( "No such class: " + c )
+
+			imports = self.parentMap[ elem ].findall( "import" )
+
+		else:
+			imports = elem.findall( "import" )
+
+		self.resolved.append( c )
+
+		for imp in imports:
+			if imp.text not in self.resolved:
+				self.__resolve( imp.text, classList )
+
+		classList.append([ c, elem ])
+
+class BotanClassResolver:
+	R = ""
+	CR = None
+
+	classMap = ""
+	flagCompress = True
+	oModeIfSizelt = 50 * 1024
+	returnHash = False
+	returnDynamic = False
+	resv = None
+
+	def __init__( self, jwork, BotanRoot, classMap, cacheRoot ):
+
+		self.JWork = jwork;
+		self.R = os.path.abspath( BotanRoot )
+		self.CR = os.path.abspath( cacheRoot )
+
+		os.makedirs( self.CR, 0o755, exist_ok = True )
+
+		if not os.path.exists( classMap ):
+			self.JWork.buildClassMap( self.R, classMap )
+
+		self.resv = Resolver( classMap )
+
+	def BotanFile( self, t ):
+		content = ""
+		with open( os.path.join( self.R, t ), "r" ) as f:
+			content = f.read()
+
+		return content
+
+	def BotanCache( self, srcFile, fileHash, hashContentDynamic ):
+
+		for _ in [0]:
+			if hashContentDynamic and os.path.getsize( srcFile ) < self.oModeIfSizelt:
+				break
+
+			if self.returnHash:
+				return fileHash
+
+		with open( srcFile, "r" ) as f:
+			return f.read()
+
+	def cleanList( self, lista ):
+		olist = []
+		for i in lista:
+			if i not in olist:
+				olist.append( i )
+		return olist
+
+	def jsLookup( self, classList, classFiles ):
+		for c in classList:
+			src = self.resv.resource( c[1] )
+
+			if src == None:
+				raise LookupError( "Cannot find src file for: "  + c[0] )
+
+			if src not in classFiles:
+				classFiles.append( src )
+
+	def cssLookup( self, classList, cssList ):
+
+		for cdef in classList:
+			f = cdef[ "src" ]
+
+			cssFile = os.path.splitext( f )[0] + ".css"
+
+			cssGroup = []
+			if not cdef[ "css" ] == "1":
+				cssGroup.append( cdef )
+
+			f = f.split( PY_SEP )
+			l = len( f )
+
+			for i in range( 1, l ):
+				key = PY_SEP.join( x for x in f[:-i] ) + PY_SEP + "_this.js"
+				_def = self.resv.locate( key )
+
+				if _def and not _def[ "css" ] == "1":
+					cssGroup.append( _def )
+
+			cssGroup.sort( key = lambda x : x[ "src" ] )
+			cssList.extend( cssGroup )
+
+	def getCache( self, fileList, cName, mode ):
+		if self.CR == None:
+			return None
+
+		md5 = hashlib.md5( bytearray( "|".join( x[mode] for x in fileList ), "utf-8" ) ).hexdigest()
+
+		cName[0] = oFHash = md5 + "." + mode
+		cFHash = md5 + ".c." + mode
+
+		# Raw file
+		oFile = os.path.join( self.CR, oFHash )
+		# Compressed file
+		cFile = os.path.join( self.CR, cFHash )
+
+		cCached = self.useCache( cFile )
+
+		if self.flagCompress and cCached:
+			return self.BotanCache( cFile, cFHash, self.returnDynamic )
+
+		if self.useCache( oFile ):
+
+			if not cCached:
+				self.JWork.saveCache(
+					oFile
+					# Content is None to initiate a compression
+					, None
+					, mode
+					, os.path.join( self.R, "externs" )
+				)
+
+			return self.BotanCache( oFile, oFHash, False )
+
+		return None
+
+	def useCache( self, f ):
+		return os.path.exists( f )
+
+	def compileJs( self, cList, xList ):
+		md5 = [ None ]
+
+		for x in xList:
+			cList.remove( x ) if x in cList else None
+
+		cacheFile = self.getCache( cList, md5, "js" )
+
+		if cacheFile is not None:
+			return cacheFile;
+
+		# The root file
+		outputJs = self.BotanFile( "_this.js" )
+
+		for f in cList:
+			f = f[ "src" ]
+			path = (
+				os.path.splitext( f )[0]
+					.replace( PY_SEP, "." )
+					.replace( "._this", "" )
+			)
+			struct = ";BotanJS.define( \"" + path + "\" );"
+
+			outputJs += struct + self.BotanFile( f )
+
+		outputJs = wrapScope( outputJs )
+
+		self.JWork.saveCache(
+			os.path.join( self.CR, md5[0] )
+			, outputJs
+			, "js"
+			, os.path.join( self.R, "externs" )
+		)
+
+		if self.returnHash:
+			return md5[0]
+
+		return outputJs
+
+	def compileCss( self, cList, xList ):
+		cssIList = []
+		cssXList = []
+		self.cssLookup( cList, cssIList )
+		self.cssLookup( xList, cssXList )
+
+		cList = []
+		xList = cssXList
+
+		for x in cssIList:
+			cList.append( x ) if x not in xList else None
+
+		md5 = [ None ]
+		cacheFile = self.getCache( cList, md5, "css" )
+
+		if cacheFile is not None:
+			return cacheFile;
+
+		struct = "/* @ */"
+		outputCss = struct + self.BotanFile( "_this.css" )
+
+		for f in self.cleanList( cList ):
+			outputCss += self.BotanFile( f["src"][:-2] + "css" )
+
+		self.JWork.saveCache( os.path.join( self.CR, md5[0] ), outputCss, "css" )
+
+		if self.returnHash:
+			return md5[0]
+
+		return outputCss
+
+	def getAPI( self, code, mode ):
+		self.flagCompress = True
+		# Should reload on debug mode only
+		self.resv._reload()
+		flag = mode[0]
+		requestAPIs = code
+
+		# Return compressed contents if possible
+		# otherwise return raw contents
+		if flag == "o":
+			mode = mode[1:]
+
+		# Return raw contents only
+		elif flag == "r":
+			mode = mode[1:]
+			self.flagCompress = False
+
+		# Return hashed filenames only
+		elif flag == "h":
+			mode = mode[1:]
+			self.returnHash = True
+
+		# Return hashed filenames if content is larger than self.oModeIfSizelt bytes
+		# otherwise act as "o" mode
+		else:
+			self.returnHash = True
+			self.returnDynamic = True
+
+		try:
+			requestAPIs = (
+				# decode -> decompress -> split
+				zlib.decompress( base64.b64decode( code, None, True ) )
+					.decode( "utf-8" )
+			)
+			sp = ","
+		except binascii.Error:
+			sp = "/"
+
+		# strip malicious
+		requestAPIs = (
+				requestAPIs
+					.replace( "[^A-Za-z\.\*" + re.escape( sp ) + " ]", "" )
+					.split( sp )
+			)
+
+		imports = []
+		excludes = []
+
+		for apis in requestAPIs:
+
+			if not apis:
+				continue
+
+			classList = []
+			lookupList = imports
+
+			if apis[0] == "-":
+				apis = apis[1:]
+				lookupList = excludes
+
+			self.resv.resolve( apis, classList )
+			self.jsLookup( classList, lookupList )
+
+		if mode == "js":
+			return self.compileJs( imports, excludes )
+		elif mode == "css":
+			return self.compileCss( imports, excludes )
+
+		raise TypeError( "Invalid mode: " + js )

botanjs/old/service/jwork.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+import os
+from botanjs.classmap import ClassMap
+
+if os.getenv( "UNIT_TEST" ) == "1":
+	CeleryExists =  False
+else:
+	CeleryExists = True
+
+try:
+	from celery import Celery
+except ImportError:
+	CeleryExists = False
+
+if CeleryExists:
+	from celery.utils.log import get_task_logger
+	app = Celery( "botanJWork" )
+	log = get_task_logger( __name__ )
+
+	if os.path.exists( "settings.ini" ):
+		from botanjs.config import Config
+		app.conf.update( broker_url = Config["BotanJS"]["CeleryBroker"] )
+
+else:
+	from botanjs.dummy import app
+	from botanjs.dummy import log
+
+class JWork:
+
+	def saveCache( location, content = None, mode = None, externs = "" ):
+		if content != None:
+			log.info( "Writing file(" +  str( len( content ) ) + "): " + os.path.abspath( location ) )
+			with open( location, "w" ) as f:
+				f.write( content )
+
+		if mode == "js":
+			JWork.compressJs.delay( location, externs )
+		elif mode == "css":
+			JWork.compressCss( location )
+
+	@app.task()
+	def compressJs( md5, externs ):
+		from botanjs.compressor.closure import Wrapper as ClosureWrapper
+		log.info( "Compress js: " + md5 )
+		w = ClosureWrapper()
+		w.scanExterns( externs )
+		w.compress( md5 )
+
+	@app.task()
+	def compressCss( md5 ):
+		from botanjs.compressor.yui import Wrapper as YUIWrapper
+		log.info( "Compress css: " + md5 )
+		w = YUIWrapper()
+		w.compress( md5 )
+
+	@app.task()
+	def buildClassMap( src, location ):
+		log.info( "Building Class Map" )
+		c = ClassMap( src )
+
+		os.makedirs( os.path.dirname( location ), exist_ok = True )
+		with open( location, "w" ) as f:
+			f.write( c.build() )

botanjs/old/service/webapi.py

@@ -0,0 +1,66 @@
+from flask import Flask
+from flask import Response
+from flask import render_template
+from flask import request
+from botanjs.service.jclassresv import BotanClassResolver as JCResv
+from botanjs.service.jwork import app as CeleryApp, JWork
+from botanjs.config import Config
+
+import os
+
+class WebAPI:
+
+	app = None
+
+	BRoot = None
+	BMap = None
+	BCache = None
+
+	def __init__( self, jsRoot = "../src", jsCache = "/tmp", brokerURL = None ):
+
+		self.BRoot = os.path.abspath( jsRoot )
+		self.BCache = os.path.join( jsCache, "botanjs" )
+		self.BMap = os.path.join( self.BCache, "bmap.xml" )
+
+		if brokerURL != None:
+			CeleryApp.conf.update( broker_url = brokerURL )
+
+		self.app = Flask( __name__, static_url_path = "/cache/botanjs", static_folder = self.BCache )
+		self.app.jinja_env.add_extension( "compressinja.html.HtmlCompressor" )
+
+		self.app.add_url_rule( "/" , view_func = self.index )
+		self.app.add_url_rule( "/<mode>/" , view_func = lambda mode: self.api_request( mode, "zpayload" ) )
+		self.app.add_url_rule( "/<mode>/<path:code>" , view_func = self.api_request )
+
+	def run( self, *args, **kwargs ):
+		JWork.buildClassMap( self.BRoot, self.BMap )
+		return self.app.run( *args, **kwargs )
+
+	def index( self ):
+		return "Hello, this is the BotanJS Service API.", 200
+
+	def api_request( self, mode, code ):
+
+		if mode == "rebuild":
+			JWork.buildClassMap.delay( self.BRoot, self.BMap )
+			return "OK", 200
+
+		if code == "zpayload":
+			code = request.args.get( "p" )
+
+		try:
+			t = mode[1:]
+			if t == "js":
+				t = "application/javascript"
+			elif t == "css":
+				t = "text/css"
+
+			srvHandler = JCResv( JWork, self.BRoot, self.BMap, self.BCache )
+			return Response( srvHandler.getAPI( code, mode = mode ), mimetype = t )
+		except Exception as e:
+			if self.app.config[ "DEBUG" ]:
+				raise
+			return str(e), 404
+
+
+		return "Invalid request", 404

botanjs/old/utils.py

@@ -0,0 +1,14 @@
+from functools import lru_cache
+from zlib import adler32 as _HashFunc
+HashFunc = lambda v: hex( _HashFunc( v ) )[2:]
+
+@lru_cache( maxsize = 1024 )
+def checksum( file_path ):
+	return checksum_r( file_path )
+
+def checksum_r( file_path ):
+	try:
+		with open( file_path, "rb" ) as f:
+			return HashFunc( f.read() )
+	except FileNotFoundError:
+		return HashFunc( b"" )