diff --git a/.drone.yml b/.drone.yml index c6f35114a2..1755783442 100644 --- a/.drone.yml +++ b/.drone.yml @@ -542,6 +542,15 @@ pipeline: when: matrix: TESTS: integration-remote-api + integration-download: + image: nextcloudci/integration-php7.0:integration-php7.0-6 + commands: + - ./occ maintenance:install --admin-pass=admin --data-dir=/dev/shm/nc_int + - cd build/integration + - ./run.sh --tags ~@large features/download.feature + when: + matrix: + TESTS: integration-download acceptance-access-levels: image: nextcloudci/integration-php7.0:integration-php7.0-6 commands: @@ -738,6 +747,7 @@ matrix: - TESTS: integration-ldap-features - TESTS: integration-trashbin - TESTS: integration-remote-api + - TESTS: integration-download - TESTS: acceptance TESTS-ACCEPTANCE: access-levels - TESTS: acceptance diff --git a/build/integration/features/bootstrap/BasicStructure.php b/build/integration/features/bootstrap/BasicStructure.php index 0aead766f2..9769037f19 100644 --- a/build/integration/features/bootstrap/BasicStructure.php +++ b/build/integration/features/bootstrap/BasicStructure.php @@ -39,6 +39,7 @@ require __DIR__ . '/../../vendor/autoload.php'; trait BasicStructure { use Auth; + use Download; use Trashbin; /** @var string */ @@ -356,8 +357,41 @@ trait BasicStructure { * @param string $text */ public function modifyTextOfFile($user, $filename, $text) { - self::removeFile("../../data/$user/files", "$filename"); - file_put_contents("../../data/$user/files" . "$filename", "$text"); + self::removeFile($this->getDataDirectory() . "/$user/files", "$filename"); + file_put_contents($this->getDataDirectory() . "/$user/files" . "$filename", "$text"); + } + + private function getDataDirectory() { + // Based on "runOcc" from CommandLine trait + $args = ['config:system:get', 'datadirectory']; + $args = array_map(function($arg) { + return escapeshellarg($arg); + }, $args); + $args[] = '--no-ansi --no-warnings'; + $args = implode(' ', $args); + + $descriptor = [ + 0 => ['pipe', 'r'], + 1 => ['pipe', 'w'], + 2 => ['pipe', 'w'], + ]; + $process = proc_open('php console.php ' . $args, $descriptor, $pipes, $ocPath = '../..'); + $lastStdOut = stream_get_contents($pipes[1]); + proc_close($process); + + return trim($lastStdOut); + } + + /** + * @Given file :filename is created :times times in :user user data + * @param string $filename + * @param string $times + * @param string $user + */ + public function fileIsCreatedTimesInUserData($filename, $times, $user) { + for ($i = 0; $i < $times; $i++) { + file_put_contents($this->getDataDirectory() . "/$user/files" . "$filename-$i", "content-$i"); + } } public function createFileSpecificSize($name, $size) { diff --git a/build/integration/features/bootstrap/Download.php b/build/integration/features/bootstrap/Download.php new file mode 100644 index 0000000000..90e2bdf67a --- /dev/null +++ b/build/integration/features/bootstrap/Download.php @@ -0,0 +1,143 @@ +. + * + */ + +use GuzzleHttp\Client; +use GuzzleHttp\Message\ResponseInterface; + +require __DIR__ . '/../../vendor/autoload.php'; + +trait Download { + + /** @var string **/ + private $downloadedFile; + + /** @AfterScenario **/ + public function cleanupDownloadedFile() { + $this->downloadedFile = null; + } + + /** + * @When user :user downloads zip file for entries :entries in folder :folder + */ + public function userDownloadsZipFileForEntriesInFolder($user, $entries, $folder) { + $this->asAn($user); + $this->sendingToDirectUrl('GET', "/index.php/apps/files/ajax/download.php?dir=" . $folder . "&files=[" . $entries . "]"); + $this->theHTTPStatusCodeShouldBe('200'); + + $this->getDownloadedFile(); + } + + private function getDownloadedFile() { + $this->downloadedFile = ''; + + $body = $this->response->getBody(); + while (!$body->eof()) { + $this->downloadedFile .= $body->read(8192); + } + $body->close(); + } + + /** + * @Then the downloaded zip file is a zip32 file + */ + public function theDownloadedZipFileIsAZip32File() { + // assertNotContains is not used to prevent the whole file from being + // printed in case of error. + PHPUnit_Framework_Assert::assertTrue( + strpos($this->downloadedFile, "\x50\x4B\x06\x06") === false, + "File contains the zip64 end of central dir signature" + ); + } + + /** + * @Then the downloaded zip file is a zip64 file + */ + public function theDownloadedZipFileIsAZip64File() { + // assertNotContains is not used to prevent the whole file from being + // printed in case of error. + PHPUnit_Framework_Assert::assertTrue( + strpos($this->downloadedFile, "\x50\x4B\x06\x06") !== false, + "File does not contain the zip64 end of central dir signature" + ); + } + + /** + * @Then the downloaded zip file contains a file named :fileName with the contents of :sourceFileName from :user data + */ + public function theDownloadedZipFileContainsAFileNamedWithTheContentsOfFromData($fileName, $sourceFileName, $user) { + $fileHeaderRegExp = '/'; + $fileHeaderRegExp .= "\x50\x4B\x03\x04"; // Local file header signature + $fileHeaderRegExp .= '.{22,22}'; // Ignore from "version needed to extract" to "uncompressed size" + $fileHeaderRegExp .= preg_quote(pack('v', strlen($fileName)), '/'); // File name length + $fileHeaderRegExp .= '(.{2,2})'; // Get "extra field length" + $fileHeaderRegExp .= preg_quote($fileName, '/'); // File name + $fileHeaderRegExp .= '/s'; // PCRE_DOTALL, so all characters (including bytes that happen to be new line characters) match + + // assertRegExp is not used to prevent the whole file from being printed + // in case of error and to be able to get the extra field length. + PHPUnit_Framework_Assert::assertEquals( + 1, preg_match($fileHeaderRegExp, $this->downloadedFile, $matches), + "Local header for file did not appear once in zip file" + ); + + $extraFieldLength = unpack('vextraFieldLength', $matches[1])['extraFieldLength']; + $expectedFileContents = file_get_contents($this->getDataDirectory() . "/$user/files" . $sourceFileName); + + $fileHeaderAndContentRegExp = '/'; + $fileHeaderAndContentRegExp .= "\x50\x4B\x03\x04"; // Local file header signature + $fileHeaderAndContentRegExp .= '.{22,22}'; // Ignore from "version needed to extract" to "uncompressed size" + $fileHeaderAndContentRegExp .= preg_quote(pack('v', strlen($fileName)), '/'); // File name length + $fileHeaderAndContentRegExp .= '.{2,2}'; // Ignore "extra field length" + $fileHeaderAndContentRegExp .= preg_quote($fileName, '/'); // File name + $fileHeaderAndContentRegExp .= '.{' . $extraFieldLength . ',' . $extraFieldLength . '}'; // Ignore "extra field" + $fileHeaderAndContentRegExp .= preg_quote($expectedFileContents, '/'); // File contents + $fileHeaderAndContentRegExp .= '/s'; // PCRE_DOTALL, so all characters (including bytes that happen to be new line characters) match + + // assertRegExp is not used to prevent the whole file from being printed + // in case of error. + PHPUnit_Framework_Assert::assertEquals( + 1, preg_match($fileHeaderAndContentRegExp, $this->downloadedFile), + "Local header and contents for file did not appear once in zip file" + ); + } + + /** + * @Then the downloaded zip file contains a folder named :folderName + */ + public function theDownloadedZipFileContainsAFolderNamed($folderName) { + $folderHeaderRegExp = '/'; + $folderHeaderRegExp .= "\x50\x4B\x03\x04"; // Local file header signature + $folderHeaderRegExp .= '.{22,22}'; // Ignore from "version needed to extract" to "uncompressed size" + $folderHeaderRegExp .= preg_quote(pack('v', strlen($folderName)), '/'); // File name length + $folderHeaderRegExp .= '.{2,2}'; // Ignore "extra field length" + $folderHeaderRegExp .= preg_quote($folderName, '/'); // File name + $folderHeaderRegExp .= '/s'; // PCRE_DOTALL, so all characters (including bytes that happen to be new line characters) match + + // assertRegExp is not used to prevent the whole file from being printed + // in case of error. + PHPUnit_Framework_Assert::assertEquals( + 1, preg_match($folderHeaderRegExp, $this->downloadedFile), + "Local header for folder did not appear once in zip file" + ); + } +} diff --git a/build/integration/features/download.feature b/build/integration/features/download.feature new file mode 100644 index 0000000000..16d346b015 --- /dev/null +++ b/build/integration/features/download.feature @@ -0,0 +1,294 @@ +Feature: download + + Scenario: downloading 2 small files returns a zip32 + Given using new dav path + And user "user0" exists + And User "user0" copies file "/welcome.txt" to "/welcome2.txt" + When user "user0" downloads zip file for entries '"welcome.txt","welcome2.txt"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/welcome.txt" from "user0" data + And the downloaded zip file contains a file named "welcome2.txt" with the contents of "/welcome2.txt" from "user0" data + + Scenario: downloading a small file and a directory returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/emptySubFolder" + When user "user0" downloads zip file for entries '"welcome.txt","emptySubFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "emptySubFolder/" + + Scenario: downloading a small file and 2 nested directories returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/subFolder" + And user "user0" created a folder "/subFolder/emptySubSubFolder" + When user "user0" downloads zip file for entries '"welcome.txt","subFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "subFolder/" + And the downloaded zip file contains a folder named "subFolder/emptySubSubFolder/" + + Scenario: downloading dir with 2 small files returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/sparseFolder" + And User "user0" copies file "/welcome.txt" to "/sparseFolder/welcome.txt" + And User "user0" copies file "/welcome.txt" to "/sparseFolder/welcome2.txt" + When user "user0" downloads zip file for entries '"sparseFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "sparseFolder/" + And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/sparseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a file named "sparseFolder/welcome2.txt" with the contents of "/sparseFolder/welcome2.txt" from "user0" data + + Scenario: downloading dir with a small file and a directory returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/sparseFolder" + And User "user0" copies file "/welcome.txt" to "/sparseFolder/welcome.txt" + And user "user0" created a folder "/sparseFolder/emptySubFolder" + When user "user0" downloads zip file for entries '"sparseFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "sparseFolder/" + And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/sparseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "sparseFolder/emptySubFolder/" + + Scenario: downloading dir with a small file and 2 nested directories returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/sparseFolder" + And User "user0" copies file "/welcome.txt" to "/sparseFolder/welcome.txt" + And user "user0" created a folder "/sparseFolder/subFolder" + And user "user0" created a folder "/sparseFolder/subFolder/emptySubSubFolder" + When user "user0" downloads zip file for entries '"sparseFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "sparseFolder/" + And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/sparseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "sparseFolder/subFolder/" + And the downloaded zip file contains a folder named "sparseFolder/subFolder/emptySubSubFolder/" + + Scenario: downloading (from folder) 2 small files returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/baseFolder" + And User "user0" copies file "/welcome.txt" to "/baseFolder/welcome.txt" + And User "user0" copies file "/welcome.txt" to "/baseFolder/welcome2.txt" + When user "user0" downloads zip file for entries '"welcome.txt","welcome2.txt"' in folder "/baseFolder/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/baseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a file named "welcome2.txt" with the contents of "/baseFolder/welcome2.txt" from "user0" data + + Scenario: downloading (from folder) a small file and a directory returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/baseFolder" + And User "user0" copies file "/welcome.txt" to "/baseFolder/welcome.txt" + And user "user0" created a folder "/baseFolder/emptySubFolder" + When user "user0" downloads zip file for entries '"welcome.txt","emptySubFolder"' in folder "/baseFolder/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/baseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "emptySubFolder/" + + Scenario: downloading (from folder) a small file and 2 nested directories returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/baseFolder" + And User "user0" copies file "/welcome.txt" to "/baseFolder/welcome.txt" + And user "user0" created a folder "/baseFolder/subFolder" + And user "user0" created a folder "/baseFolder/subFolder/emptySubSubFolder" + When user "user0" downloads zip file for entries '"welcome.txt","subFolder"' in folder "/baseFolder/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/baseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "subFolder/" + And the downloaded zip file contains a folder named "subFolder/emptySubSubFolder/" + + Scenario: downloading (from folder) dir with 2 small files returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/baseFolder" + And user "user0" created a folder "/baseFolder/sparseFolder" + And User "user0" copies file "/welcome.txt" to "/baseFolder/sparseFolder/welcome.txt" + And User "user0" copies file "/welcome.txt" to "/baseFolder/sparseFolder/welcome2.txt" + When user "user0" downloads zip file for entries '"sparseFolder"' in folder "/baseFolder/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "sparseFolder/" + And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/baseFolder/sparseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a file named "sparseFolder/welcome2.txt" with the contents of "/baseFolder/sparseFolder/welcome2.txt" from "user0" data + + Scenario: downloading (from folder) dir with a small file and a directory returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/baseFolder" + And user "user0" created a folder "/baseFolder/sparseFolder" + And User "user0" copies file "/welcome.txt" to "/baseFolder/sparseFolder/welcome.txt" + And user "user0" created a folder "/baseFolder/sparseFolder/emptySubFolder" + When user "user0" downloads zip file for entries '"sparseFolder"' in folder "/baseFolder/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "sparseFolder/" + And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/baseFolder/sparseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "sparseFolder/emptySubFolder/" + + Scenario: downloading (from folder) dir with a small file and 2 nested directories returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/baseFolder" + And user "user0" created a folder "/baseFolder/sparseFolder" + And User "user0" copies file "/welcome.txt" to "/baseFolder/sparseFolder/welcome.txt" + And user "user0" created a folder "/baseFolder/sparseFolder/subFolder" + And user "user0" created a folder "/baseFolder/sparseFolder/subFolder/emptySubSubFolder" + When user "user0" downloads zip file for entries '"sparseFolder"' in folder "/baseFolder/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "sparseFolder/" + And the downloaded zip file contains a file named "sparseFolder/welcome.txt" with the contents of "/baseFolder/sparseFolder/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "sparseFolder/subFolder/" + And the downloaded zip file contains a folder named "sparseFolder/subFolder/emptySubSubFolder/" + + @large + Scenario: downloading small file and dir with 65524 small files and 9 nested directories returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/crowdedFolder" + And user "user0" created a folder "/crowdedFolder/subFolder1" + And file "/crowdedFolder/subFolder1/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder1" + And user "user0" created a folder "/crowdedFolder/subFolder2" + And file "/crowdedFolder/subFolder2/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder2" + And user "user0" created a folder "/crowdedFolder/subFolder3" + And file "/crowdedFolder/subFolder3/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder3" + And user "user0" created a folder "/crowdedFolder/subFolder4" + And file "/crowdedFolder/subFolder4/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder4" + And user "user0" created a folder "/crowdedFolder/subFolder5" + And file "/crowdedFolder/subFolder5/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder5" + And user "user0" created a folder "/crowdedFolder/subFolder6" + And file "/crowdedFolder/subFolder6/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder6" + And user "user0" created a folder "/crowdedFolder/subFolder7" + And file "/crowdedFolder/subFolder7/test.txt" is created "5524" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder7" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder" + When user "user0" downloads zip file for entries '"welcome.txt","crowdedFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "crowdedFolder/" + And the downloaded zip file contains a folder named "crowdedFolder/subFolder1/" + And the downloaded zip file contains a file named "crowdedFolder/subFolder1/test.txt-0" with the contents of "/crowdedFolder/subFolder1/test.txt-0" from "user0" data + And the downloaded zip file contains a file named "crowdedFolder/subFolder7/test.txt-5523" with the contents of "/crowdedFolder/subFolder7/test.txt-5523" from "user0" data + And the downloaded zip file contains a folder named "crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder/" + + @large + Scenario: downloading dir with 65525 small files and 9 nested directories returns a zip32 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/crowdedFolder" + And user "user0" created a folder "/crowdedFolder/subFolder1" + And file "/crowdedFolder/subFolder1/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder1" + And user "user0" created a folder "/crowdedFolder/subFolder2" + And file "/crowdedFolder/subFolder2/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder2" + And user "user0" created a folder "/crowdedFolder/subFolder3" + And file "/crowdedFolder/subFolder3/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder3" + And user "user0" created a folder "/crowdedFolder/subFolder4" + And file "/crowdedFolder/subFolder4/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder4" + And user "user0" created a folder "/crowdedFolder/subFolder5" + And file "/crowdedFolder/subFolder5/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder5" + And user "user0" created a folder "/crowdedFolder/subFolder6" + And file "/crowdedFolder/subFolder6/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder6" + And user "user0" created a folder "/crowdedFolder/subFolder7" + And file "/crowdedFolder/subFolder7/test.txt" is created "5525" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder7" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder" + When user "user0" downloads zip file for entries '"crowdedFolder"' in folder "/" + Then the downloaded zip file is a zip32 file + And the downloaded zip file contains a folder named "crowdedFolder/" + And the downloaded zip file contains a folder named "crowdedFolder/subFolder1/" + And the downloaded zip file contains a file named "crowdedFolder/subFolder1/test.txt-0" with the contents of "/crowdedFolder/subFolder1/test.txt-0" from "user0" data + And the downloaded zip file contains a file named "crowdedFolder/subFolder7/test.txt-5524" with the contents of "/crowdedFolder/subFolder7/test.txt-5524" from "user0" data + And the downloaded zip file contains a folder named "crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder/" + + @large + Scenario: downloading small file and dir with 65524 small files and 10 nested directories returns a zip64 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/crowdedFolder" + And user "user0" created a folder "/crowdedFolder/subFolder1" + And file "/crowdedFolder/subFolder1/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder1" + And user "user0" created a folder "/crowdedFolder/subFolder2" + And file "/crowdedFolder/subFolder2/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder2" + And user "user0" created a folder "/crowdedFolder/subFolder3" + And file "/crowdedFolder/subFolder3/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder3" + And user "user0" created a folder "/crowdedFolder/subFolder4" + And file "/crowdedFolder/subFolder4/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder4" + And user "user0" created a folder "/crowdedFolder/subFolder5" + And file "/crowdedFolder/subFolder5/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder5" + And user "user0" created a folder "/crowdedFolder/subFolder6" + And file "/crowdedFolder/subFolder6/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder6" + And user "user0" created a folder "/crowdedFolder/subFolder7" + And file "/crowdedFolder/subFolder7/test.txt" is created "5524" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder7" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder" + And user "user0" created a folder "/crowdedFolder/subFolder7/emptySubSubFolder" + When user "user0" downloads zip file for entries '"welcome.txt","crowdedFolder"' in folder "/" + Then the downloaded zip file is a zip64 file + And the downloaded zip file contains a file named "welcome.txt" with the contents of "/welcome.txt" from "user0" data + And the downloaded zip file contains a folder named "crowdedFolder/" + And the downloaded zip file contains a folder named "crowdedFolder/subFolder1/" + And the downloaded zip file contains a file named "crowdedFolder/subFolder1/test.txt-0" with the contents of "/crowdedFolder/subFolder1/test.txt-0" from "user0" data + And the downloaded zip file contains a file named "crowdedFolder/subFolder7/test.txt-5523" with the contents of "/crowdedFolder/subFolder7/test.txt-5523" from "user0" data + And the downloaded zip file contains a folder named "crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder/" + And the downloaded zip file contains a folder named "crowdedFolder/subFolder7/emptySubSubFolder/" + + @large + Scenario: downloading dir with 65525 small files and 10 nested directories returns a zip64 + Given using new dav path + And user "user0" exists + And user "user0" created a folder "/crowdedFolder" + And user "user0" created a folder "/crowdedFolder/subFolder1" + And file "/crowdedFolder/subFolder1/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder1" + And user "user0" created a folder "/crowdedFolder/subFolder2" + And file "/crowdedFolder/subFolder2/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder2" + And user "user0" created a folder "/crowdedFolder/subFolder3" + And file "/crowdedFolder/subFolder3/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder3" + And user "user0" created a folder "/crowdedFolder/subFolder4" + And file "/crowdedFolder/subFolder4/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder4" + And user "user0" created a folder "/crowdedFolder/subFolder5" + And file "/crowdedFolder/subFolder5/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder5" + And user "user0" created a folder "/crowdedFolder/subFolder6" + And file "/crowdedFolder/subFolder6/test.txt" is created "10000" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder6" + And user "user0" created a folder "/crowdedFolder/subFolder7" + And file "/crowdedFolder/subFolder7/test.txt" is created "5525" times in "user0" user data + And invoking occ with "files:scan --path /user0/files/crowdedFolder/subFolder7" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder" + And user "user0" created a folder "/crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder" + And user "user0" created a folder "/crowdedFolder/subFolder7/emptySubSubFolder" + When user "user0" downloads zip file for entries '"crowdedFolder"' in folder "/" + Then the downloaded zip file is a zip64 file + And the downloaded zip file contains a folder named "crowdedFolder/" + And the downloaded zip file contains a folder named "crowdedFolder/subFolder1/" + And the downloaded zip file contains a file named "crowdedFolder/subFolder1/test.txt-0" with the contents of "/crowdedFolder/subFolder1/test.txt-0" from "user0" data + And the downloaded zip file contains a file named "crowdedFolder/subFolder7/test.txt-5524" with the contents of "/crowdedFolder/subFolder7/test.txt-5524" from "user0" data + And the downloaded zip file contains a folder named "crowdedFolder/subFolder7/subSubFolder/emptySubSubSubFolder/" + And the downloaded zip file contains a folder named "crowdedFolder/subFolder7/emptySubSubFolder/" diff --git a/build/integration/run.sh b/build/integration/run.sh index 45c2bcdaf2..b747bb52c6 100755 --- a/build/integration/run.sh +++ b/build/integration/run.sh @@ -2,6 +2,12 @@ OC_PATH=../../ OCC=${OC_PATH}occ +TAGS="" +if [ "$1" = "--tags" ]; then + TAGS="--tags=$2" + + shift 2 +fi SCENARIO_TO_RUN=$1 HIDE_OC_LOGS=$2 @@ -52,7 +58,7 @@ if [ "$INSTALLED" == "true" ]; then fi -vendor/bin/behat --strict -f junit -f pretty $SCENARIO_TO_RUN +vendor/bin/behat --strict -f junit -f pretty $TAGS $SCENARIO_TO_RUN RESULT=$? kill $PHPPID diff --git a/lib/private/Streamer.php b/lib/private/Streamer.php index 7b178fda65..51c2c923c2 100644 --- a/lib/private/Streamer.php +++ b/lib/private/Streamer.php @@ -24,6 +24,7 @@ namespace OC; +use OCP\IRequest; use ownCloud\TarStreamer\TarStreamer; use ZipStreamer\ZipStreamer; @@ -33,12 +34,42 @@ class Streamer { // streamer instance private $streamerInstance; - - public function __construct(){ - /** @var \OCP\IRequest */ - $request = \OC::$server->getRequest(); - - if ($request->isUserAgent($this->preferTarFor)) { + + /** + * Streamer constructor. + * + * @param IRequest $request + * @param int $size The size of the files in bytes + * @param int $numberOfFiles The number of files (and directories) that will + * be included in the streamed file + */ + public function __construct(IRequest $request, int $size, int $numberOfFiles){ + + /** + * zip32 constraints for a basic (without compression, volumes nor + * encryption) zip file according to the Zip specification: + * - No file size is larger than 4 bytes (file size < 4294967296); see + * 4.4.9 uncompressed size + * - The size of all files plus their local headers is not larger than + * 4 bytes; see 4.4.16 relative offset of local header and 4.4.24 + * offset of start of central directory with respect to the starting + * disk number + * - The total number of entries (files and directories) in the zip file + * is not larger than 2 bytes (number of entries < 65536); see 4.4.22 + * total number of entries in the central dir + * - The size of the central directory is not larger than 4 bytes; see + * 4.4.23 size of the central directory + * + * Due to all that, zip32 is used if the size is below 4GB and there are + * less than 65536 files; the margin between 4*1000^3 and 4*1024^3 + * should give enough room for the extra zip metadata. Technically, it + * would still be possible to create an invalid zip32 file (for example, + * a zip file from files smaller than 4GB with a central directory + * larger than 4GiB), but it should not happen in the real world. + */ + if ($size < 4 * 1000 * 1000 * 1000 && $numberOfFiles < 65536) { + $this->streamerInstance = new ZipStreamer(['zip64' => false]); + } else if ($request->isUserAgent($this->preferTarFor)) { $this->streamerInstance = new TarStreamer(); } else { $this->streamerInstance = new ZipStreamer(['zip64' => PHP_INT_SIZE !== 4]); diff --git a/lib/private/legacy/files.php b/lib/private/legacy/files.php index def9f82fab..9281c1f7da 100644 --- a/lib/private/legacy/files.php +++ b/lib/private/legacy/files.php @@ -144,17 +144,34 @@ class OC_Files { } } - $streamer = new Streamer(); - OC_Util::obEnd(); - self::lockFiles($view, $dir, $files); + /* Calculate filesize and number of files */ + if ($getType === self::ZIP_FILES) { + $fileInfos = array(); + $fileSize = 0; + foreach ($files as $file) { + $fileInfo = \OC\Files\Filesystem::getFileInfo($dir . '/' . $file); + $fileSize += $fileInfo->getSize(); + $fileInfos[] = $fileInfo; + } + $numberOfFiles = self::getNumberOfFiles($fileInfos); + } elseif ($getType === self::ZIP_DIR) { + $fileInfo = \OC\Files\Filesystem::getFileInfo($dir . '/' . $files); + $fileSize = $fileInfo->getSize(); + $numberOfFiles = self::getNumberOfFiles(array($fileInfo)); + } + + $streamer = new Streamer(\OC::$server->getRequest(), $fileSize, $numberOfFiles); + OC_Util::obEnd(); + $streamer->sendHeaders($name); $executionTime = (int)OC::$server->getIniWrapper()->getNumeric('max_execution_time'); if (strpos(@ini_get('disable_functions'), 'set_time_limit') === false) { @set_time_limit(0); } ignore_user_abort(true); + if ($getType === self::ZIP_FILES) { foreach ($files as $file) { $file = $dir . '/' . $file; @@ -313,6 +330,29 @@ class OC_Files { } } + /** + * Returns the total (recursive) number of files and folders in the given + * FileInfos. + * + * @param \OCP\Files\FileInfo[] $fileInfos the FileInfos to count + * @return int the total number of files and folders + */ + private static function getNumberOfFiles($fileInfos) { + $numberOfFiles = 0; + + $view = new View(); + + while ($fileInfo = array_pop($fileInfos)) { + $numberOfFiles++; + + if ($fileInfo->getType() === \OCP\Files\FileInfo::TYPE_FOLDER) { + $fileInfos = array_merge($fileInfos, $view->getDirectoryContent($fileInfo->getPath())); + } + } + + return $numberOfFiles; + } + /** * @param View $view * @param string $dir