Lock SCSS so we only run 1 job at a time

This is bit hacky but a start to lock the SCSS compiler properly
Retry during 10s then give up
Properly get error message
Do not clear locks and properly debug scss caching

Signed-off-by: Roeland Jago Douma <roeland@famdouma.nl>
This commit is contained in:
Roeland Jago Douma 2019-05-29 16:09:07 +02:00
parent c193c0d466
commit f8aeef7ae9
No known key found for this signature in database
GPG Key ID: F941078878347C0C
3 changed files with 79 additions and 20 deletions

View File

@ -238,6 +238,9 @@ class IconsCacher {
} }
} }
/**
* Add the icons cache css into the header
*/
public function injectCss() { public function injectCss() {
$mtime = $this->timeFactory->getTime(); $mtime = $this->timeFactory->getTime();
$file = $this->getCachedList(); $file = $this->getCachedList();

View File

@ -32,6 +32,7 @@ use Leafo\ScssPhp\Compiler;
use Leafo\ScssPhp\Exception\ParserException; use Leafo\ScssPhp\Exception\ParserException;
use Leafo\ScssPhp\Formatter\Crunched; use Leafo\ScssPhp\Formatter\Crunched;
use Leafo\ScssPhp\Formatter\Expanded; use Leafo\ScssPhp\Formatter\Expanded;
use OC\Memcache\NullCache;
use OCP\AppFramework\Utility\ITimeFactory; use OCP\AppFramework\Utility\ITimeFactory;
use OCP\Files\IAppData; use OCP\Files\IAppData;
use OCP\Files\NotFoundException; use OCP\Files\NotFoundException;
@ -42,6 +43,7 @@ use OCP\ICache;
use OCP\ICacheFactory; use OCP\ICacheFactory;
use OCP\IConfig; use OCP\IConfig;
use OCP\ILogger; use OCP\ILogger;
use OCP\IMemcache;
use OCP\IURLGenerator; use OCP\IURLGenerator;
use OC\Files\AppData\Factory; use OC\Files\AppData\Factory;
use OC\Template\IconsCacher; use OC\Template\IconsCacher;
@ -84,6 +86,9 @@ class SCSSCacher {
/** @var ITimeFactory */ /** @var ITimeFactory */
private $timeFactory; private $timeFactory;
/** @var IMemcache */
private $lockingCache;
/** /**
* @param ILogger $logger * @param ILogger $logger
* @param Factory $appDataFactory * @param Factory $appDataFactory
@ -111,8 +116,13 @@ class SCSSCacher {
$this->defaults = $defaults; $this->defaults = $defaults;
$this->serverRoot = $serverRoot; $this->serverRoot = $serverRoot;
$this->cacheFactory = $cacheFactory; $this->cacheFactory = $cacheFactory;
$this->depsCache = $cacheFactory->createDistributed('SCSS-' . md5($this->urlGenerator->getBaseUrl())); $this->depsCache = $cacheFactory->createDistributed('SCSS-deps-' . md5($this->urlGenerator->getBaseUrl()));
$this->isCachedCache = $cacheFactory->createLocal('SCSS-cached-' . md5($this->urlGenerator->getBaseUrl())); $this->isCachedCache = $cacheFactory->createLocal('SCSS-cached-' . md5($this->urlGenerator->getBaseUrl()));
$lockingCache = $cacheFactory->createDistributed('SCSS-locks-' . md5($this->urlGenerator->getBaseUrl()));
if (!($lockingCache instanceof IMemcache)) {
$lockingCache = new NullCache();
}
$this->lockingCache = $lockingCache;
$this->iconsCacher = $iconsCacher; $this->iconsCacher = $iconsCacher;
$this->timeFactory = $timeFactory; $this->timeFactory = $timeFactory;
} }
@ -137,10 +147,7 @@ class SCSSCacher {
if (!$this->variablesChanged() && $this->isCached($fileNameCSS, $app)) { if (!$this->variablesChanged() && $this->isCached($fileNameCSS, $app)) {
// Inject icons vars css if any // Inject icons vars css if any
if ($this->iconsCacher->getCachedCSS() && $this->iconsCacher->getCachedCSS()->getSize() > 0) { return $this->injectCssVariablesIfAny();
$this->iconsCacher->injectCss();
}
return true;
} }
try { try {
@ -150,7 +157,35 @@ class SCSSCacher {
$folder = $this->appData->newFolder($app); $folder = $this->appData->newFolder($app);
} }
$lockKey = $webDir . '/' . $fileNameSCSS;
if (!$this->lockingCache->add($lockKey, 'locked!', 120)) {
$retry = 0;
sleep(1);
while ($retry < 10) {
if (!$this->variablesChanged() && $this->isCached($fileNameCSS, $app)) {
// Inject icons vars css if any
$this->lockingCache->remove($lockKey);
$this->logger->debug('SCSSCacher: ' .$lockKey.' is now available after '.$retry.'s. Moving on...', ['app' => 'core']);
return $this->injectCssVariablesIfAny();
}
$this->logger->debug('SCSSCacher: scss cache file locked for '.$lockKey, ['app' => 'core']);
sleep($retry);
$retry++;
}
$this->logger->debug('SCSSCacher: Giving up scss caching for '.$lockKey, ['app' => 'core']);
return false;
}
try {
$cached = $this->cache($path, $fileNameCSS, $fileNameSCSS, $folder, $webDir); $cached = $this->cache($path, $fileNameCSS, $fileNameSCSS, $folder, $webDir);
} catch (\Exception $e) {
$this->lockingCache->remove($lockKey);
throw $e;
}
// Cleaning lock
$this->lockingCache->remove($lockKey);
// Inject icons vars css if any // Inject icons vars css if any
if ($this->iconsCacher->getCachedCSS() && $this->iconsCacher->getCachedCSS()->getSize() > 0) { if ($this->iconsCacher->getCachedCSS() && $this->iconsCacher->getCachedCSS()->getSize() > 0) {
@ -180,19 +215,24 @@ class SCSSCacher {
*/ */
private function isCached(string $fileNameCSS, string $app) { private function isCached(string $fileNameCSS, string $app) {
$key = $this->config->getSystemValue('version') . '/' . $app . '/' . $fileNameCSS; $key = $this->config->getSystemValue('version') . '/' . $app . '/' . $fileNameCSS;
if (!$this->config->getSystemValue('debug') && $cacheValue = $this->isCachedCache->get($key)) {
// If the file mtime is more recent than our cached one,
// let's consider the file is properly cached
if ($cacheValue = $this->isCachedCache->get($key)) {
if ($cacheValue > $this->timeFactory->getTime()) { if ($cacheValue > $this->timeFactory->getTime()) {
return true; return true;
} }
} }
// Creating file cache if none for further checks
try { try {
$folder = $this->appData->getFolder($app); $folder = $this->appData->getFolder($app);
} catch (NotFoundException $e) { } catch (NotFoundException $e) {
// creating css appdata folder return false;
$folder = $this->appData->newFolder($app);
} }
// Checking if file size is coherent
// and if one of the css dependency changed
try { try {
$cachedFile = $folder->getFile($fileNameCSS); $cachedFile = $folder->getFile($fileNameCSS);
if ($cachedFile->getSize() > 0) { if ($cachedFile->getSize() > 0) {
@ -201,7 +241,7 @@ class SCSSCacher {
if ($deps === null) { if ($deps === null) {
$depFile = $folder->getFile($depFileName); $depFile = $folder->getFile($depFileName);
$deps = $depFile->getContent(); $deps = $depFile->getContent();
//Set to memcache for next run // Set to memcache for next run
$this->depsCache->set($folder->getName() . '-' . $depFileName, $deps); $this->depsCache->set($folder->getName() . '-' . $depFileName, $deps);
} }
$deps = json_decode($deps, true); $deps = json_decode($deps, true);
@ -228,13 +268,11 @@ class SCSSCacher {
*/ */
private function variablesChanged(): bool { private function variablesChanged(): bool {
$injectedVariables = $this->getInjectedVariables(); $injectedVariables = $this->getInjectedVariables();
if ($this->config->getAppValue('core', 'scss.variables') !== md5($injectedVariables)) { if ($this->config->getAppValue('core', 'theming.variables') !== md5($injectedVariables)) {
$this->resetCache(); $this->resetCache();
$this->config->setAppValue('core', 'scss.variables', md5($injectedVariables)); $this->config->setAppValue('core', 'theming.variables', md5($injectedVariables));
return true; return true;
} }
return false; return false;
} }
@ -289,7 +327,7 @@ class SCSSCacher {
'@import "functions.scss";' . '@import "functions.scss";' .
'@import "' . $fileNameSCSS . '";'); '@import "' . $fileNameSCSS . '";');
} catch (ParserException $e) { } catch (ParserException $e) {
$this->logger->error($e, ['app' => 'core']); $this->logger->logException($e, ['app' => 'core']);
return false; return false;
} }
@ -327,7 +365,11 @@ class SCSSCacher {
*/ */
public function resetCache() { public function resetCache() {
$this->injectedVariables = null; $this->injectedVariables = null;
$this->cacheFactory->createDistributed('SCSS-')->clear();
// do not clear locks
$this->cacheFactory->createDistributed('SCSS-deps-')->clear();
$this->cacheFactory->createDistributed('SCSS-cached-')->clear();
$appDirectory = $this->appData->getDirectoryListing(); $appDirectory = $this->appData->getDirectoryListing();
foreach ($appDirectory as $folder) { foreach ($appDirectory as $folder) {
foreach ($folder->getDirectoryListing() as $file) { foreach ($folder->getDirectoryListing() as $file) {
@ -338,6 +380,7 @@ class SCSSCacher {
} }
} }
} }
$this->logger->debug('SCSSCacher: css cache cleared!');
} }
/** /**
@ -358,7 +401,7 @@ class SCSSCacher {
$scss->compile($variables); $scss->compile($variables);
$this->injectedVariables = $variables; $this->injectedVariables = $variables;
} catch (ParserException $e) { } catch (ParserException $e) {
$this->logger->error($e, ['app' => 'core']); $this->logger->logException($e, ['app' => 'core']);
} }
return $variables; return $variables;
@ -391,7 +434,7 @@ class SCSSCacher {
return substr($this->urlGenerator->linkToRoute('core.Css.getCss', [ return substr($this->urlGenerator->linkToRoute('core.Css.getCss', [
'fileName' => $fileName, 'fileName' => $fileName,
'appName' => $appName, 'appName' => $appName,
'v' => $this->config->getAppValue('core', 'scss.variables', '0') 'v' => $this->config->getAppValue('core', 'theming.variables', '0')
]), \strlen(\OC::$WEBROOT) + 1); ]), \strlen(\OC::$WEBROOT) + 1);
} }
@ -449,4 +492,17 @@ class SCSSCacher {
return $webRoot . substr($path, strlen($serverRoot)); return $webRoot . substr($path, strlen($serverRoot));
} }
/**
* Add the icons css cache in the header if needed
*
* @return boolean true
*/
private function injectCssVariablesIfAny() {
// Inject icons vars css if any
if ($this->iconsCacher->getCachedCSS() && $this->iconsCacher->getCachedCSS()->getSize() > 0) {
$this->iconsCacher->injectCss();
}
return true;
}
} }

View File

@ -528,10 +528,10 @@ class SCSSCacherTest extends \Test\TestCase {
->willReturn([$file]); ->willReturn([$file]);
$cache = $this->createMock(ICache::class); $cache = $this->createMock(ICache::class);
$this->cacheFactory->expects($this->once()) $this->cacheFactory->expects($this->exactly(2))
->method('createDistributed') ->method('createDistributed')
->willReturn($cache); ->willReturn($cache);
$cache->expects($this->once()) $cache->expects($this->exactly(2))
->method('clear') ->method('clear')
->with(''); ->with('');
$this->appData->expects($this->once()) $this->appData->expects($this->once())