Skip to content

Commit c93c6d1

Browse files
tomasnorretomasnorre
authored
[TASK] Refactor commands (#1127)
1 parent eebab53 commit c93c6d1

File tree

6 files changed

+137
-140
lines changed

6 files changed

+137
-140
lines changed

‎Classes/Command/BuildQueueCommand.php

Lines changed: 104 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,16 @@
4343
*/
4444
class BuildQueueCommand extends Command
4545
{
46+
public function __construct(
47+
private readonly JsonCompatibilityConverter $jsonCompatibilityConverter,
48+
private readonly EventDispatcher $eventDispatcher,
49+
private readonly QueueRepository $queueRepository,
50+
private readonly PageRepository $pageRepository,
51+
private readonly CrawlerController $crawlerController,
52+
) {
53+
parent::__construct();
54+
}
55+
4656
protected function configure(): void
4757
{
4858
$this->setDescription('Create entries in the queue that can be processed at once');
@@ -110,27 +120,21 @@ protected function configure(): void
110120
*/
111121
protected function execute(InputInterface $input, OutputInterface $output): int
112122
{
113-
/** @var JsonCompatibilityConverter $jsonCompatibilityConverter */
114-
$jsonCompatibilityConverter = GeneralUtility::makeInstance(JsonCompatibilityConverter::class);
115123
$mode = $input->getOption('mode') ?? 'queue';
116124

117125
$extensionSettings = GeneralUtility::makeInstance(
118126
ExtensionConfigurationProvider::class
119127
)->getExtensionConfiguration();
120-
$eventDispatcher = GeneralUtility::makeInstance(EventDispatcher::class);
121128

122129
/** @var CrawlerController $crawlerController */
123130
$crawlerController = GeneralUtility::makeInstance(CrawlerController::class);
124-
/** @var QueueRepository $queueRepository */
125-
$queueRepository = GeneralUtility::makeInstance(QueueRepository::class);
126-
$pageRepository = GeneralUtility::makeInstance(PageRepository::class);
127131

128132
if ($mode === 'exec') {
129133
$crawlerController->registerQueueEntriesInternallyOnly = true;
130134
}
131135

132136
$pageId = MathUtility::forceIntegerInRange((int) $input->getArgument('page'), 0);
133-
if ($pageId === 0 || empty($pageRepository->getPage($pageId))) {
137+
if ($pageId === 0 || empty($this->pageRepository->getPage($pageId))) {
134138
$message = "Page {$pageId} is not a valid page, please check you root page id and try again.";
135139
MessageUtility::addErrorMessage($message);
136140
$output->writeln("<info>{$message}</info>");
@@ -143,85 +147,24 @@ protected function execute(InputInterface $input, OutputInterface $output): int
143147
$reason = new Reason();
144148
$reason->setReason(Reason::REASON_CLI_SUBMIT);
145149
$reason->setDetailText('The cli script of the crawler added to the queue');
146-
$eventDispatcher->dispatch(new InvokeQueueChangeEvent($reason));
150+
$this->eventDispatcher->dispatch(new InvokeQueueChangeEvent($reason));
147151
}
148152

149153
if ($extensionSettings['cleanUpOldQueueEntries']) {
150-
$queueRepository->cleanUpOldQueueEntries();
154+
$this->queueRepository->cleanUpOldQueueEntries();
151155
}
152156

153-
$crawlerController->setID = GeneralUtility::md5int(microtime());
154-
$queueRows = $crawlerController->getPageTreeAndUrls(
155-
$pageId,
156-
MathUtility::forceIntegerInRange((int) $input->getOption('depth'), 0, 99),
157-
$crawlerController->getCurrentTime(),
158-
MathUtility::forceIntegerInRange((int) $input->getOption('number') ?: 30, 1, 1000),
159-
$mode === 'queue' || $mode === 'exec',
160-
$mode === 'url',
161-
[],
162-
$configurationKeys
163-
);
157+
$this->crawlerController->setID = GeneralUtility::md5int(microtime());
158+
$queueRows = $this->getQueueRows($pageId, $input, $mode, $configurationKeys);
164159

165-
// Consider a swith/match statement here, and extract the code in between.
166-
if ($mode === 'url') {
167-
$output->writeln('<info>' . implode(PHP_EOL, $crawlerController->downloadUrls) . PHP_EOL . '</info>');
168-
} elseif ($mode === 'exec') {
169-
$progressBar = new ProgressBar($output);
170-
$output->writeln('<info>Executing ' . count($crawlerController->urlList) . ' requests right away:</info>');
171-
$this->outputUrls($queueRows, $output);
172-
$output->writeln('<info>Processing</info>' . PHP_EOL);
173-
174-
foreach ($progressBar->iterate($crawlerController->queueEntries) as $queueRec) {
175-
$p = $jsonCompatibilityConverter->convert($queueRec['parameters']);
176-
if (is_bool($p)) {
177-
continue;
178-
}
179-
180-
$progressBar->clear();
181-
if (empty($p['procInstructions'][0])) {
182-
$procInstructionsString = '';
183-
} else {
184-
$procInstructionsString = ' (' . implode(',', $p['procInstructions']) . ')';
185-
}
186-
$output->writeln('<info>' . $p['url'] . $procInstructionsString . ' => ' . '</info>');
187-
$progressBar->display();
188-
189-
$result = $crawlerController->readUrlFromArray($queueRec);
190-
191-
$resultContent = $result['content'] ?? '';
192-
$requestResult = $jsonCompatibilityConverter->convert($resultContent);
193-
194-
$progressBar->clear();
195-
if (is_array($requestResult)) {
196-
$resLog = array_key_exists('log', $requestResult)
197-
&& is_array($requestResult['log']) ? chr(9) . chr(9) .
198-
implode(PHP_EOL . chr(9) . chr(9), $requestResult['log']) : '';
199-
$output->writeln('<info>OK: ' . $resLog . '</info>' . PHP_EOL);
200-
} else {
201-
$output->writeln(
202-
'<error>Error checking Crawler Result: ' . substr(
203-
(string) preg_replace('/\s+/', ' ', strip_tags((string) $resultContent)),
204-
0,
205-
30000
206-
) . '...' . PHP_EOL . '</error>' . PHP_EOL
207-
);
208-
}
209-
$progressBar->display();
210-
}
211-
$output->writeln('');
212-
} elseif ($mode === 'queue') {
213-
$output->writeln(
214-
'<info>Putting ' . count($crawlerController->urlList) . ' entries in queue:</info>' . PHP_EOL
215-
);
216-
$this->outputUrls($queueRows, $output);
217-
} else {
218-
$output->writeln(
219-
'<info>' . count(
220-
$crawlerController->urlList
221-
) . ' entries found for processing. (Use "mode" to decide action):</info>' . PHP_EOL
222-
);
223-
$this->outputUrls($queueRows, $output);
224-
}
160+
match ($mode) {
161+
'url' => $output->writeln(
162+
'<info>' . implode(PHP_EOL, $this->crawlerController->downloadUrls) . PHP_EOL . '</info>'
163+
),
164+
'exec' => $this->outputModeExec($output, $queueRows),
165+
'queue' => $this->outputModeQueue($output, $queueRows),
166+
default => $this->outputModeDefault($output, $queueRows),
167+
};
225168

226169
return Command::SUCCESS;
227170
}
@@ -248,4 +191,85 @@ private function outputUrls(array $queueRows, OutputInterface $output): void
248191
}
249192
}
250193
}
194+
195+
private function outputModeDefault(OutputInterface $output, array $queueRows): void
196+
{
197+
$output->writeln(
198+
'<info>' . count(
199+
$this->crawlerController->urlList
200+
) . ' entries found for processing. (Use "mode" to decide action):</info>' . PHP_EOL
201+
);
202+
$this->outputUrls($queueRows, $output);
203+
}
204+
205+
private function outputModeQueue(OutputInterface $output, array $queueRows): void
206+
{
207+
$output->writeln(
208+
'<info>Putting ' . count($this->crawlerController->urlList) . ' entries in queue:</info>' . PHP_EOL
209+
);
210+
$this->outputUrls($queueRows, $output);
211+
}
212+
213+
private function outputModeExec(OutputInterface $output, array $queueRows): void
214+
{
215+
$progressBar = new ProgressBar($output);
216+
$output->writeln(
217+
'<info>Executing ' . count($this->crawlerController->urlList) . ' requests right away:</info>'
218+
);
219+
$this->outputUrls($queueRows, $output);
220+
$output->writeln('<info>Processing</info>' . PHP_EOL);
221+
222+
foreach ($progressBar->iterate($this->crawlerController->queueEntries) as $queueRec) {
223+
$p = $this->jsonCompatibilityConverter->convert($queueRec['parameters']);
224+
if (is_bool($p)) {
225+
continue;
226+
}
227+
228+
$progressBar->clear();
229+
if (empty($p['procInstructions'][0])) {
230+
$procInstructionsString = '';
231+
} else {
232+
$procInstructionsString = ' (' . implode(',', $p['procInstructions']) . ')';
233+
}
234+
$output->writeln('<info>' . $p['url'] . $procInstructionsString . ' => ' . '</info>');
235+
$progressBar->display();
236+
237+
$result = $this->crawlerController->readUrlFromArray($queueRec);
238+
239+
$resultContent = $result['content'] ?? '';
240+
$requestResult = $this->jsonCompatibilityConverter->convert($resultContent);
241+
242+
$progressBar->clear();
243+
if (is_array($requestResult)) {
244+
$resLog = array_key_exists('log', $requestResult)
245+
&& is_array($requestResult['log']) ? chr(9) . chr(9) .
246+
implode(PHP_EOL . chr(9) . chr(9), $requestResult['log']) : '';
247+
$output->writeln('<info>OK: ' . $resLog . '</info>' . PHP_EOL);
248+
} else {
249+
$output->writeln(
250+
'<error>Error checking Crawler Result: ' . substr(
251+
(string) preg_replace('/\s+/', ' ', strip_tags((string) $resultContent)),
252+
0,
253+
30000
254+
) . '...' . PHP_EOL . '</error>' . PHP_EOL
255+
);
256+
}
257+
$progressBar->display();
258+
}
259+
$output->writeln('');
260+
}
261+
262+
private function getQueueRows(int $pageId, InputInterface $input, mixed $mode, array $configurationKeys): array
263+
{
264+
return $this->crawlerController->getPageTreeAndUrls(
265+
$pageId,
266+
MathUtility::forceIntegerInRange((int) $input->getOption('depth'), 0, 99),
267+
$this->crawlerController->getCurrentTime(),
268+
MathUtility::forceIntegerInRange((int) $input->getOption('number') ?: 30, 1, 1000),
269+
$mode === 'queue' || $mode === 'exec',
270+
$mode === 'url',
271+
[],
272+
$configurationKeys
273+
);
274+
}
251275
}

‎Classes/Command/ProcessQueueCommand.php

Lines changed: 11 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -41,27 +41,17 @@ class ProcessQueueCommand extends Command
4141
private const CLI_STATUS_PROCESSED = 2;
4242
private const CLI_STATUS_ABORTED = 4;
4343
private const CLI_STATUS_POLLABLE_PROCESSED = 8;
44-
45-
private Crawler $crawler;
46-
private CrawlerController $crawlerController;
47-
private ProcessRepository $processRepository;
48-
private QueueRepository $queueRepository;
4944
private string $processId;
5045
private array $extensionSettings;
5146

5247
public function __construct(
53-
Crawler $crawler,
54-
CrawlerController $crawlerController,
55-
ProcessRepository $processRepository,
56-
QueueRepository $queueRepository,
57-
string $name = null
48+
private readonly Crawler $crawler,
49+
private readonly CrawlerController $crawlerController,
50+
private readonly ProcessRepository $processRepository,
51+
private readonly QueueRepository $queueRepository,
5852
) {
59-
parent::__construct($name);
60-
$this->crawler = $crawler;
61-
$this->crawlerController = $crawlerController;
62-
$this->processRepository = $processRepository;
63-
$this->queueRepository = $queueRepository;
6453
$this->processId = md5(microtime() . random_bytes(12));
54+
parent::__construct();
6555
}
6656

6757
/**
@@ -82,15 +72,7 @@ public function execute(InputInterface $input, OutputInterface $output): int
8272

8373
$result = self::CLI_STATUS_NOTHING_PROCCESSED;
8474

85-
/** @var QueueRepository $queueRepository */
86-
$queueRepository = GeneralUtility::makeInstance(QueueRepository::class);
87-
/** @var ProcessRepository $processRepository */
88-
$processRepository = GeneralUtility::makeInstance(ProcessRepository::class);
89-
90-
/** @var Crawler $crawler */
91-
$crawler = GeneralUtility::makeInstance(Crawler::class);
92-
93-
if (!$crawler->isDisabled() && $this->checkAndAcquireNewProcess($this->processId)) {
75+
if (!$this->crawler->isDisabled() && $this->checkAndAcquireNewProcess($this->processId)) {
9476
$countInARun = $amount ? (int) $amount : (int) $this->extensionSettings['countInARun'];
9577
$sleepAfterFinish = $sleepafter ? (int) $sleepafter : (int) $this->extensionSettings['sleepAfterFinish'];
9678
$sleepTime = $sleeptime ? (int) $sleeptime : (int) $this->extensionSettings['sleepTime'];
@@ -104,17 +86,17 @@ public function execute(InputInterface $input, OutputInterface $output): int
10486
}
10587

10688
// Cleanup
107-
$processRepository->deleteProcessesWithoutItemsAssigned();
108-
$processRepository->markRequestedProcessesAsNotActive([$this->processId]);
109-
$queueRepository->unsetProcessScheduledAndProcessIdForQueueEntries([$this->processId]);
89+
$this->processRepository->deleteProcessesWithoutItemsAssigned();
90+
$this->processRepository->markRequestedProcessesAsNotActive([$this->processId]);
91+
$this->queueRepository->unsetProcessScheduledAndProcessIdForQueueEntries([$this->processId]);
11092

11193
$output->writeln(
11294
'<info>Unprocessed Items remaining:' . count(
113-
$queueRepository->getUnprocessedItems()
95+
$this->queueRepository->getUnprocessedItems()
11496
) . ' (' . $this->processId . ')</info>'
11597
);
11698
$result |= (count(
117-
$queueRepository->getUnprocessedItems()
99+
$this->queueRepository->getUnprocessedItems()
118100
) > 0 ? self::CLI_STATUS_REMAIN : self::CLI_STATUS_NOTHING_PROCCESSED);
119101
} else {
120102
$result |= self::CLI_STATUS_ABORTED;

‎Configuration/Services.yaml

Lines changed: 20 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,26 @@ services:
1212
- name: 'console.command'
1313
command: 'crawler:buildQueue'
1414
description: 'Create entries in the queue that can be processed at once'
15+
arguments:
16+
$jsonCompatibilityConverter: '@AOE\Crawler\Converter\JsonCompatibilityConverter'
17+
$eventDispatcher: '@TYPO3\CMS\Core\EventDispatcher\EventDispatcher'
18+
$queueRepository: '@AOE\Crawler\Domain\Repository\QueueRepository'
19+
$pageRepository: '@TYPO3\CMS\Core\Domain\Repository\PageRepository'
20+
$crawlerController: '@AOE\Crawler\Controller\CrawlerController'
21+
public: true
22+
23+
AOE\Crawler\Command\ProcessQueueCommand:
24+
arguments:
25+
$crawler: '@AOE\Crawler\Crawler'
26+
$crawlerController: '@AOE\Crawler\Controller\CrawlerController'
27+
$processRepository: '@AOE\Crawler\Domain\Repository\ProcessRepository'
28+
$queueRepository: '@AOE\Crawler\Domain\Repository\QueueRepository'
29+
public: true
30+
tags:
31+
- name: 'console.command'
32+
command: 'crawler:processQueue'
33+
description: 'Trigger the crawler to process the queue entries'
34+
1535

1636
AOE\Crawler\Command\FlushQueueCommand:
1737
tags:
@@ -66,18 +86,6 @@ services:
6686
$eventDispatcher: '@TYPO3\CMS\Core\EventDispatcher\EventDispatcher'
6787
public: true
6888

69-
AOE\Crawler\Command\ProcessQueueCommand:
70-
arguments:
71-
$crawler: '@AOE\Crawler\Crawler'
72-
$crawlerController: '@AOE\Crawler\Controller\CrawlerController'
73-
$processRepository: '@AOE\Crawler\Domain\Repository\ProcessRepository'
74-
$queueRepository: '@AOE\Crawler\Domain\Repository\QueueRepository'
75-
public: true
76-
tags:
77-
- name: 'console.command'
78-
command: 'crawler:processQueue'
79-
description: 'Trigger the crawler to process the queue entries'
80-
8189
AOE\Crawler\Controller\BackendModuleController:
8290
tags: [ 'backend.controller' ]
8391

‎Tests/Functional/Command/BuildQueueCommandTest.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ protected function setUp(): void
9595
);
9696

9797
$this->queueRepository = GeneralUtility::makeInstance(QueueRepository::class);
98-
$command = new BuildQueueCommand();
98+
$command = GeneralUtility::makeInstance(BuildQueueCommand::class);
9999
$this->commandTester = new CommandTester($command);
100100
}
101101

‎Tests/Functional/Command/ProcessQueueCommandTest.php

Lines changed: 1 addition & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,10 +20,6 @@
2020
*/
2121

2222
use AOE\Crawler\Command\ProcessQueueCommand;
23-
use AOE\Crawler\Controller\CrawlerController;
24-
use AOE\Crawler\Crawler;
25-
use AOE\Crawler\Domain\Repository\ProcessRepository;
26-
use AOE\Crawler\Domain\Repository\QueueRepository;
2723
use AOE\Crawler\Tests\Functional\BackendRequestTestTrait;
2824
use PHPUnit\Framework\Attributes\DataProvider;
2925
use PHPUnit\Framework\Attributes\Test;
@@ -47,14 +43,7 @@ protected function setUp(): void
4743
$this->importCSVDataSet(__DIR__ . '/../Fixtures/tx_crawler_queue.csv');
4844
$this->importCSVDataSet(__DIR__ . '/../Fixtures/pages.csv');
4945

50-
$crawlerController = GeneralUtility::makeInstance(CrawlerController::class);
51-
52-
$command = new ProcessQueueCommand(
53-
new Crawler(),
54-
$crawlerController,
55-
GeneralUtility::makeInstance(ProcessRepository::class),
56-
GeneralUtility::makeInstance(QueueRepository::class)
57-
);
46+
$command = GeneralUtility::makeInstance(ProcessQueueCommand::class);
5847
$this->commandTester = new CommandTester($command);
5948
}
6049

‎phpstan-baseline.neon

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,5 @@
11
parameters:
22
ignoreErrors:
3-
-
4-
message: '#^Cognitive complexity for "AOE\\Crawler\\Command\\BuildQueueCommand\:\:execute\(\)" is 23, keep it under 21$#'
5-
identifier: complexity.functionLike
6-
count: 1
7-
path: Classes/Command/BuildQueueCommand.php
8-
93
-
104
message: '#^Constant AOE\\Crawler\\Command\\ProcessQueueCommand\:\:CLI_STATUS_POLLABLE_PROCESSED is unused\.$#'
115
identifier: classConstant.unused

0 commit comments

Comments
 (0)