Commit 01271156 authored by Thomas Löffler's avatar Thomas Löffler

Merge branch 'develop' into 'main'

Release 30-11-2020

See merge request t3o/ter!625
parents 001e53e0 34a1901b
Pipeline #10109 passed with stages
in 11 minutes and 56 seconds
#!/bin/bash
## Description: Generate ecdsa keys for generating tokens in the REST API
## Usage: create-keys
## Example: "ddev create-keys"
readonly KEYS_PATH=/var/www/html
mkdir -p $KEYS_PATH
openssl ecparam -name secp521r1 -genkey -noout -out $KEYS_PATH/private.test.ec.key
openssl pkcs8 -topk8 -in $KEYS_PATH/private.test.ec.key -out $KEYS_PATH/private.test.pem -passin env:ECDSA_KEY_PASSPHRASE -passout env:ECDSA_KEY_PASSPHRASE
openssl ec -in $KEYS_PATH/private.test.pem -pubout -out $KEYS_PATH/public.test.pem -passin env:ECDSA_KEY_PASSPHRASE
rm $KEYS_PATH/private.test.ec.key
......@@ -8,3 +8,11 @@ services:
- DB_HOST=db
- DB_NAME=db
- DB_PASSWORD=db
- ECDSA_PUBLIC_KEY_FILE=/var/www/html/public.test.pem
- ECDSA_PRIVATE_KEY_FILE=/var/www/html/private.test.pem
- ECDSA_KEY_PASSPHRASE=passphrase
- TER_REST_RANDOM_LENGTH=30
- TER_REST_DEFAULT_LIFETIME=604800
- TER_REST_JWT_SUBJECT=t3o-ter-rest
- TER_REST_JWT_LATENCY=60
- TER_REST_SIGNATURE_IDENTIFIER=ecdsa
......@@ -9,6 +9,7 @@
!/public/.well-known/security.txt
/auth.json
/assets/
/*.pem
sequelpro.spf
.php_cs.cache
.ddev/db_snapshots/
......
......@@ -32,13 +32,14 @@ test:unit:
- apt-get update -yqq
- apt-get install git unzip zlib1g-dev libzip-dev -yqq
- docker-php-ext-install zip
- pecl install xdebug
- pecl install xdebug-2.9.2
- docker-php-ext-enable xdebug
- curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer
- export TYPO3_PATH_WEB="$PWD/private"
- composer config cache-dir /cache/composer
- mkdir $TYPO3_PATH_WEB/fileadmin/ && touch $TYPO3_PATH_WEB/fileadmin/currentcoredata.json
script:
- composer selfupdate --1
- composer install --ignore-platform-reqs
- composer test:unit
artifacts:
......@@ -57,13 +58,14 @@ test:mutation:
- apt-get update -yqq
- apt-get install git unzip zlib1g-dev libzip-dev -yqq
- docker-php-ext-install zip
- pecl install xdebug
- pecl install xdebug-2.9.2
- docker-php-ext-enable xdebug
- curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer
- export TYPO3_PATH_WEB="$PWD/private"
- composer config cache-dir /cache/composer
- mkdir $TYPO3_PATH_WEB/fileadmin/ && touch $TYPO3_PATH_WEB/fileadmin/currentcoredata.json
script:
- composer selfupdate --1
- composer install --ignore-platform-reqs
- ./vendor/bin/infection --min-msi=10 --min-covered-msi=75 --threads=4
artifacts:
......@@ -97,4 +99,4 @@ mutation:badge:
paths:
- badges/
when: always
expire_in: 4 weeks
\ No newline at end of file
expire_in: 4 weeks
......@@ -17,10 +17,14 @@
<testsuite name="ter_fe2 tests">
<directory>../../extensions/ter_fe2/Tests/Unit</directory>
</testsuite>
<testsuite name="ter_rest tests">
<directory>../../extensions/ter_rest/Tests/Unit</directory>
</testsuite>
</testsuites>
<filter>
<whitelist processUncoveredFilesFromWhitelist="true">
<directory suffix=".php">../../extensions/ter_fe2/Classes/</directory>
<directory suffix=".php">../../extensions/ter_rest/Classes/</directory>
</whitelist>
</filter>
<logging>
......
......@@ -31,6 +31,7 @@
],
"require": {
"php": "^7.4",
"ext-curl": "*",
"ext-dom": "*",
"ext-json": "*",
"ext-pdo": "*",
......@@ -39,10 +40,12 @@
"apache-solr-for-typo3/solr": "^11.0",
"cweagans/composer-patches": "^1.7",
"gordalina/cachetool": "^4.0",
"lcobucci/jwt": "^3.3",
"nimut/testing-framework": "^5.0",
"t3o/t3olayout": "^5.0",
"t3o/ter-frontend": "^0.5.0",
"t3o/ter-layout": "^0.2.0",
"t3o/ter-rest": "^0.1.0",
"t3o/ter-soap": "^2.1",
"typo3/cms-adminpanel": "^10.4",
"typo3/cms-seo": "^10.4"
......@@ -51,6 +54,7 @@
"codeception/codeception": "^4.1",
"codeception/module-asserts": "^1.2",
"codeception/module-phpbrowser": "^1.0",
"friendsofphp/php-cs-fixer": "^2.16",
"infection/infection": "^0.20.0",
"neos/utility-arrays": "^6.0",
"neos/utility-files": "^6.0",
......@@ -83,6 +87,9 @@
],
"test:api": [
"./vendor/bin/codecept run api --steps"
],
"test:php": [
"./vendor/bin/php-cs-fixer fix --dry-run --diff-format=udiff --path-mode=intersection --config=.gitlab-ci/build/.php_cs --diff extensions"
]
}
}
This diff is collapsed.
<?php
declare(strict_types = 1);
namespace T3o\TerFe2\Command;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use T3o\TerFe2\Domain\Model\Extension;
use T3o\TerFe2\Domain\Repository\ExtensionRepository;
use T3o\TerFe2\Domain\Repository\VersionRepository;
use T3o\TerFe2\Utility\VersionUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManager;
use TYPO3\CMS\Extbase\Persistence\Generic\PersistenceManager;
use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
class PackagistCommand extends Command
{
/**
* @var ObjectManager
*/
protected $objectManager;
/**
* @var ExtensionRepository
*/
protected $extensionRepository;
/**
* @var VersionRepository
*/
protected $versionRepository;
public function __construct(string $name = null, ObjectManager $objectManager = null, ExtensionRepository $extensionRepository = null, VersionRepository $versionRepository = null)
{
$this->objectManager = $objectManager ?? GeneralUtility::makeInstance(ObjectManager::class);
$this->extensionRepository = $extensionRepository ?? $this->objectManager->get(ExtensionRepository::class);
$this->versionRepository = $versionRepository ?? $this->objectManager->get(VersionRepository::class);
parent::__construct($name);
}
protected function configure()
{
$this->setDescription('Fetches download numbers from Packagist');
$this->addOption(
'limit',
'l',
InputArgument::OPTIONAL,
'This will set the limit of how many extension to fetch download data off',
25
);
}
/**
* Fetch download data from Packagist
*
* @param InputInterface $input
* @param OutputInterface $output
*
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\IllegalObjectTypeException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\InvalidQueryException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\UnknownObjectException
*/
public function execute(InputInterface $input, OutputInterface $output)
{
$objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$io = new SymfonyStyle($input, $output);
$limit = (int)$input->getOption('limit');
$querySettings = $objectManager->get(Typo3QuerySettings::class);
$querySettings->setRespectStoragePage(false);
$this->extensionRepository->setDefaultQuerySettings($querySettings);
$this->versionRepository->setDefaultQuerySettings($querySettings);
$extensions = $this->extensionRepository->findWithComposerName($limit);
// User-Agent https://twitter.com/seldaek/status/1095420243315511297
$options = ['http' => ['user_agent' => 'Thanks a bunch! Fetching packagist Download Data for https://extensions.typo3.org - https://typo3.org/community/teams/typo3org/']];
$context = stream_context_create($options);
/** @var Extension $extension */
foreach ($extensions as $extension) {
$io->writeln('<info> Processing: ' . $extension->getExtKey() . '</info>');
$packagistUrl = 'https://packagist.org/packages/' . $extension->getComposerName() . '/downloads.json';
/** @var Extension $extension */
$extension = $this->extensionRepository->findOneByExtKey($extension->getExtKey());
$this->updateLastDownloadSyncForExtension($extension);
// A Packagist URL returned 404 and would proceed, so it will be skipped for this run.
// TODO: Perhaps an report should be generated if it keeps happening?
if (!$this->doesUrlReturnOKCode($packagistUrl)) {
continue;
}
$downloadData = json_decode(file_get_contents($packagistUrl, false, $context), true);
foreach ($downloadData['package']['downloads']['versions'] as $version => $downloads) {
$versionFound = $this->versionRepository->findOneByExtensionAndVersionString($extension, $version);
if (null === $versionFound) {
continue;
}
if (!$version) {
continue;
}
VersionUtility::updateVersionCounter(
$extension->getExtKey(),
$versionFound->getUid(),
VersionUtility::DOWNLOAD_SOURCE_PACKAGIST,
$downloads['monthly']
);
}
}
$io->writeln('Fetching download numbers from Packagist is completed');
}
/**
* Sets Extension Last Download Sync to secure oldest syncs happens first.
* @param Extension $extension
* @throws \TYPO3\CMS\Extbase\Object\Exception
*/
private function updateLastDownloadSyncForExtension(Extension $extension): void
{
$extension->setLastDownloadSync(time());
$this->extensionRepository->update($extension);
/** @var PersistenceManager $persistenceManager */
$persistenceManager = $this->objectManager->get(PersistenceManager::class);
$persistenceManager->persistAll();
}
private function doesUrlReturnOKCode(string $url): bool
{
$handle = curl_init($url);
curl_setopt($handle, CURLOPT_RETURNTRANSFER, true);
/* Get the HTML or whatever is linked in $url. */
$response = curl_exec($handle);
/* Check for 404 (file not found). */
$httpCode = curl_getinfo($handle, CURLINFO_HTTP_CODE);
if ($httpCode != 200) {
return false;
}
curl_close($handle);
return true;
}
}
......@@ -20,6 +20,7 @@ use T3o\Ter\Api\ExtensionVersion;
use T3o\TerFe2\Domain\Model\Extension;
use T3o\TerFe2\Provider\FileProvider;
use T3o\TerFe2\Utility\ExtensionUtility;
use T3o\TerFe2\Utility\VersionUtility;
use T3o\TerFe2\Validation\Validator\ComposerNameValidator;
use TYPO3\CMS\Core\Page\PageRenderer;
use TYPO3\CMS\Core\Utility\GeneralUtility;
......@@ -49,6 +50,11 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
*/
protected $ownerRepository;
/**
* @var \T3o\TerFe2\Domain\Repository\DownloadRepository
*/
protected $downloadRepository;
/**
* @var \T3o\TerFe2\Persistence\Session
*/
......@@ -104,6 +110,11 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
$this->ownerRepository = $ownerRepository;
}
public function injectDownloadRepository(\T3o\TerFe2\Domain\Repository\DownloadRepository $downloadRepository)
{
$this->downloadRepository = $downloadRepository;
}
/**
* inject session
*
......@@ -367,6 +378,8 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
*/
public function downloadAction(\T3o\TerFe2\Domain\Model\Extension $extension, $versionString = '', $format = '')
{
$downloadSource = VersionUtility::DOWNLOAD_SOURCE_TER;
if ($format === '') {
$format = 'zip';
}
......@@ -412,6 +425,7 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
// Check file hash of t3x packages
if ($format === 't3x') {
$downloadSource = VersionUtility::DOWNLOAD_SOURCE_EM;
$fileHash = \T3o\TerFe2\Utility\FileUtility::getFileHash($fileUrl);
if ($fileHash != $version->getFileHash()) {
$this->redirectWithMessage($this->translate('msg.file_hash_not_equal'), 'show', '', \TYPO3\CMS\Core\Messaging\FlashMessage::ERROR, null, null, ['extension' => $extension]);
......@@ -422,12 +436,8 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
if (!empty($this->settings['countDownloads'])) {
$extensionKey = $extension->getExtKey();
$downloads = $this->session->get('downloads');
$versionRepository = $this->objectManager->get(\T3o\TerFe2\Domain\Repository\VersionRepository::class);
if (empty($downloads) || !in_array($extensionKey, $downloads)) {
// Add +1 to download counter and save immediately
$version->incrementDownloadCounter();
$versionRepository->update($version);
$this->persistenceManager->persistAll();
VersionUtility::updateVersionCounter($extensionKey, $version->getUid(), $downloadSource);
// Add extension key to session
$downloads[] = $extensionKey;
......
<?php
declare(strict_types = 1);
namespace T3o\TerFe2\Controller;
/*
* This file is part of TYPO3 CMS-extension "ter_fe2", created by Oliver Bartsch.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*/
use GuzzleHttp\Exception\RequestException;
use Psr\Log\LoggerAwareInterface;
use Psr\Log\LoggerAwareTrait;
use T3o\TerFe2\Domain\DTO\TokenFormData;
use T3o\TerFe2\Domain\Repository\ExtensionRepository;
use T3o\TerFe2\Service\TokenRequestService;
use TYPO3\CMS\Core\Context\Context;
use TYPO3\CMS\Core\Log\LogManager;
use TYPO3\CMS\Core\Messaging\AbstractMessage;
use TYPO3\CMS\Extbase\Mvc\Controller\ActionController;
use TYPO3\CMS\Extbase\Property\TypeConverter\DateTimeConverter;
/**
* Controller for managing access tokens
*/
class TokenController extends ActionController implements LoggerAwareInterface
{
use LoggerAwareTrait;
private const API_ACTIONS = ['create', 'refresh', 'revoke'];
protected ExtensionRepository $extensionRepository;
protected TokenRequestService $tokenRequestService;
protected Context $context;
public function __construct(
ExtensionRepository $extensionRepository,
TokenRequestService $tokenRequestService,
Context $context,
LogManager $logManager
) {
$this->extensionRepository = $extensionRepository;
$this->tokenRequestService = $tokenRequestService;
$this->context = $context;
$this->setLogger($logManager->getLogger('TER.API.REST.GUI'));
}
public function indexAction(): void
{
$this->view->assignMultiple([
'actions' => self::API_ACTIONS,
'activeTab' => $this->getActiveTab(),
'extensions' => $this->extensionRepository->findByFrontendUser(
(string)$this->context->getPropertyFromAspect('frontend.user', 'username')
)
]);
}
/**
* @param TokenFormData $tokenFormData
* @TYPO3\CMS\Extbase\Annotation\Validate("T3o\TerFe2\Validation\Validator\TokenCreationValidator", param="tokenFormData")
*/
public function createAction(TokenFormData $tokenFormData): void
{
try {
$responseContent = (string)$this->tokenRequestService->request(
'/token',
$tokenFormData->getPassword(),
['query' => $this->tokenRequestService->createQueryArguments($tokenFormData)]
)->getBody()->getContents();
} catch (RequestException $e) {
$this->addFlashMessageForException($e);
$this->redirectToIndexAction();
}
$this->view->assign('tokenData', json_decode($responseContent, true, 512, JSON_THROW_ON_ERROR));
}
/**
* @param TokenFormData $tokenFormData
* @TYPO3\CMS\Extbase\Annotation\Validate("T3o\TerFe2\Validation\Validator\TokenUpdateValidator", param="tokenFormData")
*/
public function refreshAction(TokenFormData $tokenFormData): void
{
try {
$responseContent = (string)$this->tokenRequestService->request(
'/token/refresh',
$tokenFormData->getPassword(),
['form_params' => $this->tokenRequestService->createFormParams($tokenFormData)]
)->getBody()->getContents();
} catch (RequestException $e) {
$this->addFlashMessageForException($e);
$this->redirectToIndexAction();
}
$this->view->assign('tokenData', json_decode($responseContent, true, 512, JSON_THROW_ON_ERROR));
}
/**
* @param TokenFormData $tokenFormData
* @TYPO3\CMS\Extbase\Annotation\Validate("T3o\TerFe2\Validation\Validator\TokenUpdateValidator", param="tokenFormData")
*/
public function revokeAction(TokenFormData $tokenFormData): void
{
try {
$this->tokenRequestService->request(
'/token/revoke',
$tokenFormData->getPassword(),
['form_params' => $this->tokenRequestService->createFormParams($tokenFormData)]
);
$this->addFlashMessage('', 'Token sucessfully revoked');
} catch (RequestException $e) {
$this->addFlashMessageForException($e);
}
$this->redirectToIndexAction();
}
protected function initializeCreateAction(): void
{
if ($this->arguments->hasArgument('tokenFormData')) {
$this->arguments
->getArgument('tokenFormData')
->getPropertyMappingConfiguration()
->forProperty('expires')
->setTypeConverterOption(DateTimeConverter::class, DateTimeConverter::CONFIGURATION_DATE_FORMAT, 'Y-m-d');
}
}
protected function getActiveTab(): string
{
$activeTab = (string)($this->request->getArguments()['activeTab'] ?? '');
$apiActions = self::API_ACTIONS;
if ($activeTab === '' || !in_array($activeTab, $apiActions, true)) {
$this->redirectToIndexAction(array_shift($apiActions));
}