Merge pull request #26250 from nextcloud/backport/25529/stable20

[stable20] Fix non LGC glyphs in avatars and txt file previews
This commit is contained in:
Morris Jobke 2021-03-24 14:18:19 +01:00 committed by GitHub
commit 75871f76ad
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 584 additions and 0 deletions

View File

@ -0,0 +1,230 @@
diff --git a/nototools/merge_noto.py b/nototools/merge_noto.py
index 17c07ed..029845a 100755
--- a/nototools/merge_noto.py
+++ b/nototools/merge_noto.py
@@ -34,7 +34,7 @@ def make_puncless_font_name(script):
return make_font_name(script).replace(" ", "").replace("-", "")
-def make_font_file_name(script, weight, directory="individual/unhinted"):
+def make_font_file_name(script, weight, directory="individual/hinted"):
filename = "%s/%s-%s.ttf" % (directory, make_puncless_font_name(script), weight)
return filename
@@ -85,6 +85,11 @@ SCRIPT_TO_OPENTYPE_SCRIPT_TAG = {
"Cuneiform": "xsux",
"Cypriot": "cprt",
"Yi": "yi ",
+ "AnatolianHieroglyphs":"hluw",
+ "Bamum": "bamu",
+ "NewTaiLue": "talu",
+ "Tagbanwa": "tagb",
+ "Thaana": "thaa",
}
@@ -135,96 +140,129 @@ def add_gsub_to_font(fontfile):
def main():
merge_table = {
- "Historic": [
+ # Use a single file with all the fonts copied from merge_fonts.py.
+ "": [ # LGC,
+ "Adlam",
+ "AdlamUnjoined",
+ "AnatolianHieroglyphs",
+ "Arabic",
+ "ArabicUI",
+ "Armenian",
"Avestan",
- "Carian",
- "Egyptian Hieroglyphs",
- "Imperial Aramaic",
- "Pahlavi", # Should be 'Inscriptional Pahlavi',
- "Parthian", # Should be 'Inscriptional Parthian',
- "Linear B",
- "Lycian",
- "Lydian",
- "Mandaic",
- "Old Persian",
- "Old South Arabian",
- "Old Turkic",
- "Osmanya",
- "Phags-Pa",
- "Phoenician",
- "Samaritan",
- "Sumero-Akkadian Cuneiform",
- "Ugaritic",
- ],
- "South Asian": [
- "Devanagari",
+ "Balinese",
+ "Bamum",
+ "Batak",
"Bengali",
- "Gurmukhi",
- "Gujarati",
- "Oriya",
- "Tamil",
- "Telugu",
- "Kannada",
- "Malayalam",
- "Sinhala",
- "Thaana",
+ "BengaliUI",
"Brahmi",
- "Kaithi",
- "Kharoshthi", # Move to Historic?
- "Lepcha",
- "Limbu",
- "Meetei Mayek",
- "Ol Chiki",
- "Saurashtra",
- "Syloti Nagri",
- ],
- "Southeast Asian": [
- "Thai",
- "Lao",
- "Khmer",
- "Batak",
"Buginese",
"Buhid",
+ "CJKjp-Regular.otf",
+ "CJKkr-Regular.otf",
+ "CJKsc-Regular.otf",
+ "CJKtc-Regular.otf",
+ "CanadianAboriginal",
+ "Carian",
+ "Chakma",
"Cham",
- "Hanunoo",
- "Javanese",
- "Kayah Li",
- "New Tai Lue",
- "Rejang",
- "Sundanese",
- "Tagalog",
- "Tagbanwa",
- "Tai Le",
- "Tai Tham",
- "Tai Viet",
- ],
- "": [ # LGC,
- "Armenian",
- "Bamum",
- "Canadian Aboriginal",
"Cherokee",
"Coptic",
- "Cypriot Syllabary",
+ "Cuneiform",
+ "Cypriot",
"Deseret",
+ "Devanagari",
+ "DevanagariUI",
+ "Display",
+ "EgyptianHieroglyphs",
"Ethiopic",
"Georgian",
"Glagolitic",
"Gothic",
+ "Gujarati",
+ "GujaratiUI",
+ "Gurmukhi",
+ "GurmukhiUI",
+ "Hanunoo",
"Hebrew",
+ "ImperialAramaic",
+ "InscriptionalPahlavi",
+ "InscriptionalParthian",
+ "Javanese",
+ "Kaithi",
+ "Kannada",
+ "KannadaUI",
+ "KayahLi",
+ "Kharoshthi",
+ "Khmer",
+ "KhmerUI",
+ "Lao",
+ "LaoUI",
+ "Lepcha",
+ "Limbu",
+ "LinearB",
"Lisu",
+ "Lycian",
+ "Lydian",
+ "Malayalam",
+ "MalayalamUI",
+ "Mandaic",
+ "MeeteiMayek",
+ #"NotoSansMongolian",
+ "Mono",
+ "MonoCJKjp-Regular.otf",
+ "MonoCJKkr-Regular.otf",
+ "MonoCJKsc-Regular.otf",
+ "MonoCJKtc-Regular.otf",
+ "Myanmar",
+ "MyanmarUI",
"NKo",
+ "NewTaiLue",
"Ogham",
- "Old Italic",
+ "OlChiki",
+ "OldItalic",
+ "OldPersian",
+ "OldSouthArabian",
+ "OldTurkic",
+ "Oriya",
+ "OriyaUI",
+ "Osage",
+ "Osmanya",
+ "PhagsPa",
+ "Phoenician",
+ "Rejang",
"Runic",
+ "Samaritan",
+ "Saurashtra",
"Shavian",
+ "Sinhala",
+ "SinhalaUI",
+ "Sundanese",
+ "SylotiNagri",
+ "Symbols",
+ "Symbols2",
+ "SyriacEastern",
+ "SyriacEstrangela",
+ "SyriacWestern",
+ "Tagalog",
+ "Tagbanwa",
+ "TaiLe",
+ "TaiTham",
+ "TaiViet",
+ "Tamil",
+ "TamilUI",
+ "Telugu",
+ "TeluguUI",
+ "Thaana",
+ "Thai",
+ "ThaiUI",
+ "Tibetan",
"Tifinagh",
+ "Ugaritic",
"Vai",
+ "Yi",
],
}
- add_ui_alternative(merge_table, "South Asian")
- add_ui_alternative(merge_table, "Southeast Asian")
-
for merge_target in sorted(merge_table):
for weight in ["Regular", "Bold"]:
merger = merge.Merger()
@@ -261,7 +299,7 @@ def main():
name_record.string = name.encode("UTF-16BE")
font.save(
- make_font_file_name(merge_target, weight, directory="combined/unhinted")
+ make_font_file_name(merge_target, weight, directory="combined/hinted")
)

144
build/merge-font-noto.sh Executable file
View File

@ -0,0 +1,144 @@
#!/usr/bin/env bash
# @copyright Copyright (c) 2021, Daniel Calviño Sánchez (danxuliu@gmail.com)
#
# @license GNU AGPL version 3 or any later version
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Helper script to merge several Noto fonts in a single TTF file.
#
# The "Noto Sans" font (https://www.google.com/get/noto) only includes a subset
# of all the available glyphs in the Noto fonts. This scripts uses
# "merge_noto.py" from the Noto Tools package to add other scripts, like Arabic,
# Devanagari or Hebrew.
#
# "merge_noto.py" originally merges the fonts by region. However it was adjusted
# to merge "all" the fonts in a single file, like done by "merge_fonts.py". The
# reason to use "merge_noto.py" instead of "merge_fonts.py" is that
# "merge_noto.py" merges regular and bold fonts, which are both needed in
# Nextcloud. "merge_fonts.py" only merges regular fonts, and adjusting it to
# handle bold fonts too would have been more work than adjusting
# "merge_noto.py".
#
# Please note that, due to technical limitations of the TTF format (a single
# file can not have more than 65535 glyphs) the merged file does not include any
# Chinese, Japanese or Korean glyph (the Noto CJK files already use all the
# slots). In fact, it seems that it can not include either all the glyphs from
# all the non CJK Noto fonts, so it merges only those predefined in the
# "merge_fonts.py" script (as it is a larger set than the original one in
# "merge_noto.py").
#
# Also please note that merging the fonts is a slow process and it can take a
# while (from minutes to hours, depending on the system).
#
# To perform its job, the script requires the "docker" command to be available.
#
# The Docker Command Line Interface (the "docker" command) requires special
# permissions to talk to the Docker daemon, and those permissions are typically
# available only to the root user. Please see the Docker documentation to find
# out how to give access to a regular user to the Docker daemon:
# https://docs.docker.com/engine/installation/linux/linux-postinstall/
#
# Note, however, that being able to communicate with the Docker daemon is the
# same as being able to get root privileges for the system. Therefore, you must
# give access to the Docker daemon (and thus run this script as) ONLY to trusted
# and secure users:
# https://docs.docker.com/engine/security/security/#docker-daemon-attack-surface
# Stops the container started by this script.
function cleanUp() {
# Disable (yes, "+" disables) exiting immediately on errors to ensure that
# all the cleanup commands are executed (well, no errors should occur during
# the cleanup anyway, but just in case).
set +o errexit
echo "Cleaning up"
docker rm --volumes --force $DOCKER_CONTAINER_ID
}
# Exit immediately on errors.
set -o errexit
# Execute cleanUp when the script exits, either normally or due to an error.
trap cleanUp EXIT
# Ensure working directory is script directory, as some actions (like copying
# the patches to the container) expect that.
cd "$(dirname $0)"
# python:3.9 can not be used, as one of the requeriments of Noto Tools
# (pyclipper) fails to build.
#
# The container exits immediately if no command is given, so a Bash session
# is created to prevent that.
DOCKER_CONTAINER_ID=`docker run --rm --detach --interactive --tty python:3.8-slim bash`
# Install required dependencies.
docker exec $DOCKER_CONTAINER_ID apt-get update
docker exec $DOCKER_CONTAINER_ID apt-get install -y git gcc g++ libjpeg-dev zlib1g-dev wget
# Install Noto Tools in the container.
docker exec --workdir /tmp $DOCKER_CONTAINER_ID git clone https://github.com/googlefonts/nototools
docker exec --workdir /tmp/nototools $DOCKER_CONTAINER_ID git checkout 76b29f8f8f9b
docker exec --workdir /tmp/nototools $DOCKER_CONTAINER_ID pip install --requirement requirements.txt
docker exec --workdir /tmp/nototools $DOCKER_CONTAINER_ID pip install --editable .
# As Noto Tools were installed as "editable" the scripts can be patched after
# installation.
docker cp merge-font-noto-fix-merging-v20201206-phase3-76b29f8f8f9b.patch $DOCKER_CONTAINER_ID:/tmp/nototools/merge-font-noto-fix-merging-v20201206-phase3-76b29f8f8f9b.patch
docker exec --workdir /tmp/nototools --interactive $DOCKER_CONTAINER_ID patch --strip 1 < merge-font-noto-fix-merging-v20201206-phase3-76b29f8f8f9b.patch
# Get Noto fonts.
#
# Phase 2 Noto fonts use 2048 units per em, while phase 3 Noto fonts use 1000*.
# Currently the fonts in the released package** (apparently from 2017-10-25) are
# a mix of both, but fonts with different units per em can not be merged***.
# However, the fonts in the Git repository, although not released yet, are all
# using 1000 units per em already, so those are the ones merged.
#
# *https://github.com/googlefonts/noto-fonts/issues/908#issuecomment-298687906.
# **https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip
# ***https://fonttools.readthedocs.io/en/latest/merge.html
docker exec --workdir /tmp $DOCKER_CONTAINER_ID wget https://github.com/googlefonts/noto-fonts/archive/v20201206-phase3.tar.gz
docker exec --workdir /tmp $DOCKER_CONTAINER_ID tar -xzf v20201206-phase3.tar.gz
# noto-fonts in Git and snapshots of Git (like the package used) have a
# subdirectory for each font, but "merge_noto.py" expects to find all the fonts
# in a single directory, so the structure needs to be "flattened".
#
# Hinted fonts* adapt better to being rendered in different sizes. The full
# package in https://www.google.com/get/noto/ includes only unhinted fonts
# (according to its name**, I have not actually verified the fonts themselves),
# while the individual fonts listed below in the page are a mix of hinted and
# unhinted fonts. However, the Git directory has hinted versions of all fonts,
# so those are the ones merged (maybe there is a good reason not to merge hinted
# fonts, but seems to work :-P).
#
# *https://en.wikipedia.org/wiki/Font_hinting
# **https://noto-website-2.storage.googleapis.com/pkgs/Noto-unhinted.zip
docker exec --workdir /tmp $DOCKER_CONTAINER_ID mkdir --parent individual/hinted
docker exec --workdir /tmp $DOCKER_CONTAINER_ID find noto-fonts-20201206-phase3/hinted/ttf -iname "NotoSans*Regular.ttf" -exec mv {} individual/hinted/ \;
docker exec --workdir /tmp $DOCKER_CONTAINER_ID find noto-fonts-20201206-phase3/hinted/ttf -iname "NotoSans*Bold.ttf" -exec mv {} individual/hinted/ \;
# Merge the fonts.
docker exec --workdir /tmp $DOCKER_CONTAINER_ID mkdir --parent combined/hinted
docker exec --workdir /tmp $DOCKER_CONTAINER_ID merge_noto.py
# Copy resulting files.
#
# Noto fonts, as well as the merged files, are licensed under the SIL Open Font
# License: https://scripts.sil.org/OFL
docker cp $DOCKER_CONTAINER_ID:/tmp/combined/hinted/NotoSans-Regular.ttf ../core/fonts/NotoSans-Regular.ttf
docker cp $DOCKER_CONTAINER_ID:/tmp/combined/hinted/NotoSans-Bold.ttf ../core/fonts/NotoSans-Bold.ttf

View File

@ -2989,6 +2989,11 @@
<code>section</code>
</UndefinedInterfaceMethod>
</file>
<file src="core/Command/Preview/ResetRenderedTexts.php">
<InvalidReturnStatement occurrences="1">
<code>[]</code>
</InvalidReturnStatement>
</file>
<file src="core/Command/Upgrade.php">
<InvalidScalarArgument occurrences="11">
<code>0</code>

View File

@ -0,0 +1,202 @@
<?php
declare(strict_types=1);
/**
* @copyright Copyright (c) 2021, Daniel Calviño Sánchez <danxuliu@gmail.com>
*
* @author Daniel Calviño Sánchez <danxuliu@gmail.com>
*
* @license GNU AGPL version 3 or any later version
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
*/
namespace OC\Core\Command\Preview;
use OC\Preview\Storage\Root;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\IMimeTypeLoader;
use OCP\Files\NotFoundException;
use OCP\Files\NotPermittedException;
use OCP\IAvatarManager;
use OCP\IDBConnection;
use OCP\IUserManager;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class ResetRenderedTexts extends Command {
/** @var IDBConnection */
protected $connection;
/** @var IUserManager */
protected $userManager;
/** @var IAvatarManager */
protected $avatarManager;
/** @var Root */
private $previewFolder;
/** @var IMimeTypeLoader */
private $mimeTypeLoader;
public function __construct(IDBConnection $connection,
IUserManager $userManager,
IAvatarManager $avatarManager,
Root $previewFolder,
IMimeTypeLoader $mimeTypeLoader) {
parent::__construct();
$this->connection = $connection;
$this->userManager = $userManager;
$this->avatarManager = $avatarManager;
$this->previewFolder = $previewFolder;
$this->mimeTypeLoader = $mimeTypeLoader;
}
protected function configure() {
$this
->setName('preview:reset-rendered-texts')
->setDescription('Deletes all generated avatars and previews of text and md files')
->addOption('dry', 'd', InputOption::VALUE_NONE, 'Dry mode - will not delete any files - in combination with the verbose mode one could check the operations.');
}
protected function execute(InputInterface $input, OutputInterface $output): int {
$dryMode = $input->getOption('dry');
if ($dryMode) {
$output->writeln('INFO: The command is run in dry mode and will not modify anything.');
$output->writeln('');
}
$this->deleteAvatars($output, $dryMode);
$this->deletePreviews($output, $dryMode);
return 0;
}
private function deleteAvatars(OutputInterface $output, bool $dryMode): void {
$avatarsToDeleteCount = 0;
foreach ($this->getAvatarsToDelete() as [$userId, $avatar]) {
$output->writeln('Deleting avatar for ' . $userId, OutputInterface::VERBOSITY_VERBOSE);
$avatarsToDeleteCount++;
if ($dryMode) {
continue;
}
try {
$avatar->remove();
} catch (NotFoundException $e) {
// continue
} catch (NotPermittedException $e) {
// continue
}
}
$output->writeln('Deleted ' . $avatarsToDeleteCount . ' avatars');
$output->writeln('');
}
private function getAvatarsToDelete(): \Iterator {
foreach ($this->userManager->search('') as $user) {
$avatar = $this->avatarManager->getAvatar($user->getUID());
if (!$avatar->isCustomAvatar()) {
yield [$user->getUID(), $avatar];
}
}
}
private function deletePreviews(OutputInterface $output, bool $dryMode): void {
$previewsToDeleteCount = 0;
foreach ($this->getPreviewsToDelete() as ['name' => $previewFileId, 'path' => $filePath]) {
$output->writeln('Deleting previews for ' . $filePath, OutputInterface::VERBOSITY_VERBOSE);
$previewsToDeleteCount++;
if ($dryMode) {
continue;
}
try {
$preview = $this->previewFolder->getFolder((string)$previewFileId);
$preview->delete();
} catch (NotFoundException $e) {
// continue
} catch (NotPermittedException $e) {
// continue
}
}
$output->writeln('Deleted ' . $previewsToDeleteCount . ' previews');
}
// Copy pasted and adjusted from
// "lib/private/Preview/BackgroundCleanupJob.php".
private function getPreviewsToDelete(): \Iterator {
$qb = $this->connection->getQueryBuilder();
$qb->select('path', 'mimetype')
->from('filecache')
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($this->previewFolder->getId())));
$cursor = $qb->execute();
$data = $cursor->fetch();
$cursor->closeCursor();
if ($data === null) {
return [];
}
/*
* This lovely like is the result of the way the new previews are stored
* We take the md5 of the name (fileid) and split the first 7 chars. That way
* there are not a gazillion files in the root of the preview appdata.
*/
$like = $this->connection->escapeLikeParameter($data['path']) . '/_/_/_/_/_/_/_/%';
$qb = $this->connection->getQueryBuilder();
$qb->select('a.name', 'b.path')
->from('filecache', 'a')
->leftJoin('a', 'filecache', 'b', $qb->expr()->eq(
$qb->expr()->castColumn('a.name', IQueryBuilder::PARAM_INT), 'b.fileid'
))
->where(
$qb->expr()->andX(
$qb->expr()->like('a.path', $qb->createNamedParameter($like)),
$qb->expr()->eq('a.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('httpd/unix-directory'))),
$qb->expr()->orX(
$qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/plain'))),
$qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/markdown'))),
$qb->expr()->eq('b.mimetype', $qb->createNamedParameter($this->mimeTypeLoader->getId('text/x-markdown')))
)
)
);
$cursor = $qb->execute();
while ($row = $cursor->fetch()) {
yield $row;
}
$cursor->closeCursor();
}
}

Binary file not shown.

Binary file not shown.

View File

@ -166,6 +166,7 @@ if (\OC::$server->getConfig()->getSystemValue('installed', false)) {
));
$application->add(\OC::$server->query(\OC\Core\Command\Preview\Repair::class));
$application->add(\OC::$server->query(\OC\Core\Command\Preview\ResetRenderedTexts::class));
$application->add(new OC\Core\Command\User\Add(\OC::$server->getUserManager(), \OC::$server->getGroupManager()));
$application->add(new OC\Core\Command\User\Delete(\OC::$server->getUserManager()));

View File

@ -839,6 +839,7 @@ return array(
'OC\\Core\\Command\\Maintenance\\UpdateHtaccess' => $baseDir . '/core/Command/Maintenance/UpdateHtaccess.php',
'OC\\Core\\Command\\Maintenance\\UpdateTheme' => $baseDir . '/core/Command/Maintenance/UpdateTheme.php',
'OC\\Core\\Command\\Preview\\Repair' => $baseDir . '/core/Command/Preview/Repair.php',
'OC\\Core\\Command\\Preview\\ResetRenderedTexts' => $baseDir . '/core/Command/Preview/ResetRenderedTexts.php',
'OC\\Core\\Command\\Security\\ImportCertificate' => $baseDir . '/core/Command/Security/ImportCertificate.php',
'OC\\Core\\Command\\Security\\ListCertificates' => $baseDir . '/core/Command/Security/ListCertificates.php',
'OC\\Core\\Command\\Security\\RemoveCertificate' => $baseDir . '/core/Command/Security/RemoveCertificate.php',

View File

@ -868,6 +868,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c
'OC\\Core\\Command\\Maintenance\\UpdateHtaccess' => __DIR__ . '/../../..' . '/core/Command/Maintenance/UpdateHtaccess.php',
'OC\\Core\\Command\\Maintenance\\UpdateTheme' => __DIR__ . '/../../..' . '/core/Command/Maintenance/UpdateTheme.php',
'OC\\Core\\Command\\Preview\\Repair' => __DIR__ . '/../../..' . '/core/Command/Preview/Repair.php',
'OC\\Core\\Command\\Preview\\ResetRenderedTexts' => __DIR__ . '/../../..' . '/core/Command/Preview/ResetRenderedTexts.php',
'OC\\Core\\Command\\Security\\ImportCertificate' => __DIR__ . '/../../..' . '/core/Command/Security/ImportCertificate.php',
'OC\\Core\\Command\\Security\\ListCertificates' => __DIR__ . '/../../..' . '/core/Command/Security/ListCertificates.php',
'OC\\Core\\Command\\Security\\RemoveCertificate' => __DIR__ . '/../../..' . '/core/Command/Security/RemoveCertificate.php',

Binary file not shown.

Before

Width:  |  Height:  |  Size: 244 B

After

Width:  |  Height:  |  Size: 269 B