Commit 34a1901b authored by Thomas Löffler's avatar Thomas Löffler

Merge branch '164-download-counter-also-count-downloads-via-packagist-api' into 'develop'

Resolve "Download counter: Also count downloads via Packagist API"

Closes #164

See merge request t3o/ter!396
parents bd2c831a 9a5512ca
Pipeline #10107 passed with stages
in 9 minutes and 59 seconds
......@@ -31,6 +31,7 @@
],
"require": {
"php": "^7.4",
"ext-curl": "*",
"ext-dom": "*",
"ext-json": "*",
"ext-pdo": "*",
......
<?php
declare(strict_types = 1);
namespace T3o\TerFe2\Command;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputArgument;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Style\SymfonyStyle;
use T3o\TerFe2\Domain\Model\Extension;
use T3o\TerFe2\Domain\Repository\ExtensionRepository;
use T3o\TerFe2\Domain\Repository\VersionRepository;
use T3o\TerFe2\Utility\VersionUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManager;
use TYPO3\CMS\Extbase\Persistence\Generic\PersistenceManager;
use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
class PackagistCommand extends Command
{
/**
* @var ObjectManager
*/
protected $objectManager;
/**
* @var ExtensionRepository
*/
protected $extensionRepository;
/**
* @var VersionRepository
*/
protected $versionRepository;
public function __construct(string $name = null, ObjectManager $objectManager = null, ExtensionRepository $extensionRepository = null, VersionRepository $versionRepository = null)
{
$this->objectManager = $objectManager ?? GeneralUtility::makeInstance(ObjectManager::class);
$this->extensionRepository = $extensionRepository ?? $this->objectManager->get(ExtensionRepository::class);
$this->versionRepository = $versionRepository ?? $this->objectManager->get(VersionRepository::class);
parent::__construct($name);
}
protected function configure()
{
$this->setDescription('Fetches download numbers from Packagist');
$this->addOption(
'limit',
'l',
InputArgument::OPTIONAL,
'This will set the limit of how many extension to fetch download data off',
25
);
}
/**
* Fetch download data from Packagist
*
* @param InputInterface $input
* @param OutputInterface $output
*
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\IllegalObjectTypeException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\InvalidQueryException
* @throws \TYPO3\CMS\Extbase\Persistence\Exception\UnknownObjectException
*/
public function execute(InputInterface $input, OutputInterface $output)
{
$objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$io = new SymfonyStyle($input, $output);
$limit = (int)$input->getOption('limit');
$querySettings = $objectManager->get(Typo3QuerySettings::class);
$querySettings->setRespectStoragePage(false);
$this->extensionRepository->setDefaultQuerySettings($querySettings);
$this->versionRepository->setDefaultQuerySettings($querySettings);
$extensions = $this->extensionRepository->findWithComposerName($limit);
// User-Agent https://twitter.com/seldaek/status/1095420243315511297
$options = ['http' => ['user_agent' => 'Thanks a bunch! Fetching packagist Download Data for https://extensions.typo3.org - https://typo3.org/community/teams/typo3org/']];
$context = stream_context_create($options);
/** @var Extension $extension */
foreach ($extensions as $extension) {
$io->writeln('<info> Processing: ' . $extension->getExtKey() . '</info>');
$packagistUrl = 'https://packagist.org/packages/' . $extension->getComposerName() . '/downloads.json';
/** @var Extension $extension */
$extension = $this->extensionRepository->findOneByExtKey($extension->getExtKey());
$this->updateLastDownloadSyncForExtension($extension);
// A Packagist URL returned 404 and would proceed, so it will be skipped for this run.
// TODO: Perhaps an report should be generated if it keeps happening?
if (!$this->doesUrlReturnOKCode($packagistUrl)) {
continue;
}
$downloadData = json_decode(file_get_contents($packagistUrl, false, $context), true);
foreach ($downloadData['package']['downloads']['versions'] as $version => $downloads) {
$versionFound = $this->versionRepository->findOneByExtensionAndVersionString($extension, $version);
if (null === $versionFound) {
continue;
}
if (!$version) {
continue;
}
VersionUtility::updateVersionCounter(
$extension->getExtKey(),
$versionFound->getUid(),
VersionUtility::DOWNLOAD_SOURCE_PACKAGIST,
$downloads['monthly']
);
}
}
$io->writeln('Fetching download numbers from Packagist is completed');
}
/**
* Sets Extension Last Download Sync to secure oldest syncs happens first.
* @param Extension $extension
* @throws \TYPO3\CMS\Extbase\Object\Exception
*/
private function updateLastDownloadSyncForExtension(Extension $extension): void
{
$extension->setLastDownloadSync(time());
$this->extensionRepository->update($extension);
/** @var PersistenceManager $persistenceManager */
$persistenceManager = $this->objectManager->get(PersistenceManager::class);
$persistenceManager->persistAll();
}
private function doesUrlReturnOKCode(string $url): bool
{
$handle = curl_init($url);
curl_setopt($handle, CURLOPT_RETURNTRANSFER, true);
/* Get the HTML or whatever is linked in $url. */
$response = curl_exec($handle);
/* Check for 404 (file not found). */
$httpCode = curl_getinfo($handle, CURLINFO_HTTP_CODE);
if ($httpCode != 200) {
return false;
}
curl_close($handle);
return true;
}
}
......@@ -20,6 +20,7 @@ use T3o\Ter\Api\ExtensionVersion;
use T3o\TerFe2\Domain\Model\Extension;
use T3o\TerFe2\Provider\FileProvider;
use T3o\TerFe2\Utility\ExtensionUtility;
use T3o\TerFe2\Utility\VersionUtility;
use T3o\TerFe2\Validation\Validator\ComposerNameValidator;
use TYPO3\CMS\Core\Page\PageRenderer;
use TYPO3\CMS\Core\Utility\GeneralUtility;
......@@ -49,6 +50,11 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
*/
protected $ownerRepository;
/**
* @var \T3o\TerFe2\Domain\Repository\DownloadRepository
*/
protected $downloadRepository;
/**
* @var \T3o\TerFe2\Persistence\Session
*/
......@@ -104,6 +110,11 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
$this->ownerRepository = $ownerRepository;
}
public function injectDownloadRepository(\T3o\TerFe2\Domain\Repository\DownloadRepository $downloadRepository)
{
$this->downloadRepository = $downloadRepository;
}
/**
* inject session
*
......@@ -367,6 +378,8 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
*/
public function downloadAction(\T3o\TerFe2\Domain\Model\Extension $extension, $versionString = '', $format = '')
{
$downloadSource = VersionUtility::DOWNLOAD_SOURCE_TER;
if ($format === '') {
$format = 'zip';
}
......@@ -412,6 +425,7 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
// Check file hash of t3x packages
if ($format === 't3x') {
$downloadSource = VersionUtility::DOWNLOAD_SOURCE_EM;
$fileHash = \T3o\TerFe2\Utility\FileUtility::getFileHash($fileUrl);
if ($fileHash != $version->getFileHash()) {
$this->redirectWithMessage($this->translate('msg.file_hash_not_equal'), 'show', '', \TYPO3\CMS\Core\Messaging\FlashMessage::ERROR, null, null, ['extension' => $extension]);
......@@ -422,12 +436,8 @@ class ExtensionController extends \T3o\TerFe2\Controller\AbstractController
if (!empty($this->settings['countDownloads'])) {
$extensionKey = $extension->getExtKey();
$downloads = $this->session->get('downloads');
$versionRepository = $this->objectManager->get(\T3o\TerFe2\Domain\Repository\VersionRepository::class);
if (empty($downloads) || !in_array($extensionKey, $downloads)) {
// Add +1 to download counter and save immediately
$version->incrementDownloadCounter();
$versionRepository->update($version);
$this->persistenceManager->persistAll();
VersionUtility::updateVersionCounter($extensionKey, $version->getUid(), $downloadSource);
// Add extension key to session
$downloads[] = $extensionKey;
......
<?php
declare(strict_types = 1);
namespace T3o\TerFe2\Domain\Model;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use TYPO3\CMS\Extbase\DomainObject\AbstractEntity;
class Download extends AbstractEntity
{
/**
* @var string
*/
protected $extensionKey = '';
/**
* @var int
*/
protected $versionId = 0;
/**
* @var int
*/
protected $month = 0;
/**
* @var int
*/
protected $counter = 0;
/**
* @var int
*/
protected $source = 0;
/**
* @return string
*/
public function getExtensionKey(): string
{
return $this->extensionKey;
}
/**
* @param string $extensionKey
*/
public function setExtensionKey(string $extensionKey)
{
$this->extensionKey = $extensionKey;
}
/**
* @return int
*/
public function getVersionId(): int
{
return $this->versionId;
}
/**
* @param int $versionId
*/
public function setVersionId(int $versionId)
{
$this->versionId = $versionId;
}
/**
* @return int
*/
public function getMonth(): int
{
return $this->month;
}
/**
* @param int $month
*/
public function setMonth(int $month)
{
$this->month = $month;
}
/**
* @return int
*/
public function getCounter(): int
{
return $this->counter;
}
/**
* @param int $counter
*/
public function setCounter(int $counter)
{
$this->counter = $counter;
}
/**
* @return int
*/
public function getSource(): int
{
return $this->source;
}
/**
* @param int $source
*/
public function setSource(int $source)
{
$this->source = $source;
}
}
<?php
namespace T3o\TerFe2\Domain\Model;
/*
......@@ -14,6 +15,7 @@ namespace T3o\TerFe2\Domain\Model;
* The TYPO3 project - inspiring people to share!
*/
use T3o\TerFe2\Domain\Repository\DownloadRepository;
use T3o\TerFe2\Utility\VersionUtility;
use T3o\TerRest\DTO\Extension as ExtensionDto;
use TYPO3\CMS\Core\Utility\GeneralUtility;
......@@ -145,6 +147,21 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity implement
*/
protected $crowdinKey = '';
/**
* @var int
*/
protected $lastDownloadSync;
/**
* @var DownloadRepository
*/
protected $downloadRepository;
public function injectDownloadRepository(DownloadRepository $downloadRepository)
{
$this->downloadRepository = $downloadRepository;
}
/**
* Initialize all ObjectStorage instances.
*/
......@@ -511,6 +528,13 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity implement
* @return int All downloads
*/
public function getDownloads(): int
{
$downloads = $this->downloadRepository->findDownloadsByExtensionkey($this->getExtKey());
return $this->downloads + $downloads;
}
public function getOldDownloads(): int
{
return $this->downloads;
}
......@@ -589,6 +613,40 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity implement
return json_encode($versions);
}
/**
* @return string
*/
public function getDownloadsByTimeIntervalAsJson(): string
{
if (empty($this->versions)) {
return '';
}
$intervals = [];
// The for loop is turned around to ensure the sorting order in the multi dimensional array.
for ($i = 5; $i >= 0; $i--) {
$time = strtotime('-' . $i . 'month');
$month = date('Ym', $time);
$downloadsPackagist = $this->downloadRepository->findDownloadCounterByMonthAndExtensionKeyAndSource($month, $this->getExtKey(), VersionUtility::DOWNLOAD_SOURCE_PACKAGIST);
$downloadsTer = $this->downloadRepository->findDownloadCounterByMonthAndExtensionKeyAndSource($month, $this->getExtKey(), VersionUtility::DOWNLOAD_SOURCE_TER);
$downloadsEm = $this->downloadRepository->findDownloadCounterByMonthAndExtensionKeyAndSource($month, $this->getExtKey(), VersionUtility::DOWNLOAD_SOURCE_EM);
// How to define the start date for the feature?
if ($downloadsPackagist === 0 && $downloadsTer === 0 && $downloadsEm === 0) {
continue;
}
$intervals['release'][] = date('M. Y', $time);
$intervals['downloadsPackagist'][] = $downloadsPackagist;
$intervals['downloadsTER'][] = $downloadsTer;
$intervals['downloadsEM'][] = $downloadsEm;
$intervals['versions'][] = '';
}
return json_encode($intervals);
}
/**
* @return \DateTime|null
*/
......@@ -782,4 +840,20 @@ class Extension extends \TYPO3\CMS\Extbase\DomainObject\AbstractEntity implement
{
return new ExtensionDto($this);
}
/**
* @return int
*/
public function getLastDownloadSync(): int
{
return $this->lastDownloadSync;
}
/**
* @param int $lastDownloadSync
*/
public function setLastDownloadSync(int $lastDownloadSync)
{
$this->lastDownloadSync = $lastDownloadSync;
}
}
......@@ -56,6 +56,7 @@ class CombinedExtensionRepository
'v.review_state',
'v.em_category',
'v.download_counter',
'v.frontend_download_counter',
'v.file_hash',
'v.upload_date',
'v.upload_comment',
......@@ -72,19 +73,30 @@ class CombinedExtensionRepository
'e',
$queryBuilder->expr()->eq('v.extension', $queryBuilder->quoteIdentifier('e.uid'))
)
->leftJoin(
'v',
'tx_terfe2_domain_model_download',
'd',
$queryBuilder->expr()->eq('v.uid', $queryBuilder->quoteIdentifier('d.version_id'))
)
->where(
$queryBuilder->expr()->eq('e.hidden', 0),
$queryBuilder->expr()->eq('e.deleted', 0),
$queryBuilder->expr()->eq('v.hidden', 0),
$queryBuilder->expr()->eq('v.deleted', 0)
)
->groupBy('v.uid')
->add(
'select',
'COUNT(' . $queryBuilder->quoteIdentifier('d.counter') . ') AS newDownloads',
true
)
->execute();
while ($row = $stmt->fetch()) {
$extensionKey = $row['extensionkey'];
$versionNumber = $row['version'];
$groupedVersionsByExtension[$extensionKey]['frontend_user'] = $row['frontend_user'];
$groupedVersionsByExtension[$extensionKey]['downloads'] = $row['downloads'];
if (empty($row['external_manual'])) {
try {
......@@ -93,6 +105,8 @@ class CombinedExtensionRepository
$row['external_manual'] = '';
}
}
$row['download_counter'] += $row['frontend_download_counter'] + $row['newDownloads'];
$groupedVersionsByExtension[$extensionKey]['downloads'] += $row['download_counter'];
$groupedVersionsByExtension[$extensionKey]['versions'][$versionNumber] = $row;
}
......
<?php
declare(strict_types = 1);
namespace T3o\TerFe2\Domain\Repository;
/*
* This file is part of the TYPO3 CMS project.
*
* It is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License, either version 2
* of the License, or any later version.
*
* For the full copyright and license information, please read the
* LICENSE.txt file that was distributed with this source code.
*
* The TYPO3 project - inspiring people to share!
*/
use T3o\TerFe2\Domain\Model\Download;
use TYPO3\CMS\Core\Database\ConnectionPool;
use TYPO3\CMS\Core\Database\Query\QueryBuilder;
use TYPO3\CMS\Core\Utility\ClassNamingUtility;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Extbase\Object\ObjectManager;
use TYPO3\CMS\Extbase\Persistence\Generic\Typo3QuerySettings;
class DownloadRepository extends AbstractRepository
{
/**
* @var string
*/
protected $tableName = 'tx_terfe2_domain_model_download';
/**
* @var QueryBuilder
*/
protected $queryBuilder = QueryBuilder::class;
/**
* QueueRepository constructor.
*/
public function __construct()
{
$this->queryBuilder = GeneralUtility::makeInstance(ConnectionPool::class)->getQueryBuilderForTable($this->tableName);
$this->objectManager = GeneralUtility::makeInstance(ObjectManager::class);
$this->objectType = ClassNamingUtility::translateRepositoryNameToModelName($this->getRepositoryClassName());
}
public function findDownloadsByExtensionkey($extensionKey)
{
$total = 0;
$querySettings = GeneralUtility::makeInstance(Typo3QuerySettings::class);
$querySettings->setRespectStoragePage(false);
$query = $this->createQuery();
$query->setQuerySettings($querySettings);
$query->matching(
$query->equals('extensionKey', $extensionKey),
);
$downloads = $query->execute()->toArray();
/** @var Download $download */
foreach ($downloads as $download) {
$total = $total + $download->getCounter();