TypoScriptParser.php

Go to the documentation of this file.

<?php
namespace TYPO3\CMS\Core\TypoScript\Parser;

/*
 * This file is part of the TYPO3 CMS project.
 *
 * It is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License, either version 2
 * of the License, or any later version.
 *
 * For the full copyright and license information, please read the
 * LICENSE.txt file that was distributed with this source code.
 *
 * The TYPO3 project - inspiring people to share!
 */

use TYPO3\CMS\Core\TimeTracker\TimeTracker;
use TYPO3\CMS\Core\Utility\GeneralUtility;
use TYPO3\CMS\Core\Utility\MathUtility;
use TYPO3\CMS\Core\Utility\PathUtility;
use TYPO3\CMS\Core\Utility\StringUtility;
use TYPO3\CMS\Core\TypoScript\ExtendedTemplateService;
use TYPO3\CMS\Backend\Configuration\TypoScript\ConditionMatching\ConditionMatcher;

class TypoScriptParser
{
    public $strict = true;

    public $setup = array();

    public $raw;

    public $rawP;

    public $lastComment = '';

    public $commentSet = false;

    public $multiLineEnabled = false;

    public $multiLineObject = '';

    public $multiLineValue = array();

    public $inBrace = 0;

    public $lastConditionTrue = true;

    public $sections = array();

    public $sectionsMatch = array();

    public $syntaxHighLight = false;

    public $highLightData = array();

    public $highLightData_bracelevel = array();

    public $regComments = false;

    public $regLinenumbers = false;

    public $errors = array();

    public $lineNumberOffset = 0;

    public $breakPointLN = 0;

    public $highLightStyles = array(
        'prespace' => array('<span class="ts-prespace">', '</span>'),
        // Space before any content on a line
        'objstr_postspace' => array('<span class="ts-objstr_postspace">', '</span>'),
        // Space after the object string on a line
        'operator_postspace' => array('<span class="ts-operator_postspace">', '</span>'),
        // Space after the operator on a line
        'operator' => array('<span class="ts-operator">', '</span>'),
        // The operator char
        'value' => array('<span class="ts-value">', '</span>'),
        // The value of a line
        'objstr' => array('<span class="ts-objstr">', '</span>'),
        // The object string of a line
        'value_copy' => array('<span class="ts-value_copy">', '</span>'),
        // The value when the copy syntax (<) is used; that means the object reference
        'value_unset' => array('<span class="ts-value_unset">', '</span>'),
        // The value when an object is unset. Should not exist.
        'ignored' => array('<span class="ts-ignored">', '</span>'),
        // The "rest" of a line which will be ignored.
        'default' => array('<span class="ts-default">', '</span>'),
        // The default style if none other is applied.
        'comment' => array('<span class="ts-comment">', '</span>'),
        // Comment lines
        'condition' => array('<span class="ts-condition">', '</span>'),
        // Conditions
        'error' => array('<span class="ts-error">', '</span>'),
        // Error messages
        'linenum' => array('<span class="ts-linenum">', '</span>')
    );

    public $highLightBlockStyles = '';

    public $highLightBlockStyles_basecolor = '#cccccc';

    public $parentObject;

    public function parse($string, $matchObj = '')
    {
        $this->raw = explode(LF, $string);
        $this->rawP = 0;
        $pre = '[GLOBAL]';
        while ($pre) {
            if ($this->breakPointLN && $pre === '[_BREAK]') {
                $this->error('Breakpoint at ' . ($this->lineNumberOffset + $this->rawP - 2) . ': Line content was "' . $this->raw[($this->rawP - 2)] . '"', 1);
                break;
            }
            $preUppercase = strtoupper($pre);
            if ($pre[0] === '[' &&
                ($preUppercase === '[GLOBAL]' ||
                    $preUppercase === '[END]' ||
                    !$this->lastConditionTrue && $preUppercase === '[ELSE]')
            ) {
                $pre = trim($this->parseSub($this->setup));
                $this->lastConditionTrue = 1;
            } else {
                // We're in a specific section. Therefore we log this section
                $specificSection = $preUppercase !== '[ELSE]';
                if ($specificSection) {
                    $this->sections[md5($pre)] = $pre;
                }
                if (is_object($matchObj) && $matchObj->match($pre) || $this->syntaxHighLight) {
                    if ($specificSection) {
                        $this->sectionsMatch[md5($pre)] = $pre;
                    }
                    $pre = trim($this->parseSub($this->setup));
                    $this->lastConditionTrue = 1;
                } else {
                    $pre = $this->nextDivider();
                    $this->lastConditionTrue = 0;
                }
            }
        }
        if ($this->inBrace) {
            $this->error('Line ' . ($this->lineNumberOffset + $this->rawP - 1) . ': The script is short of ' . $this->inBrace . ' end brace(s)', 1);
        }
        if ($this->multiLineEnabled) {
            $this->error('Line ' . ($this->lineNumberOffset + $this->rawP - 1) . ': A multiline value section is not ended with a parenthesis!', 1);
        }
        $this->lineNumberOffset += count($this->raw) + 1;
    }

    public function nextDivider()
    {
        while (isset($this->raw[$this->rawP])) {
            $line = trim($this->raw[$this->rawP]);
            $this->rawP++;
            if ($line && $line[0] === '[') {
                return $line;
            }
        }
        return '';
    }

    public function parseSub(array &$setup)
    {
        while (isset($this->raw[$this->rawP])) {
            $line = ltrim($this->raw[$this->rawP]);
            $lineP = $this->rawP;
            $this->rawP++;
            if ($this->syntaxHighLight) {
                $this->regHighLight('prespace', $lineP, strlen($line));
            }
            // Breakpoint?
            // By adding 1 we get that line processed
            if ($this->breakPointLN && $this->lineNumberOffset + $this->rawP - 1 === $this->breakPointLN + 1) {
                return '[_BREAK]';
            }
            // Set comment flag?
            if (!$this->multiLineEnabled && strpos($line, '/*') === 0) {
                $this->commentSet = 1;
            }
            // If $this->multiLineEnabled we will go and get the line values here because we know, the first if() will be TRUE.
            if (!$this->commentSet && ($line || $this->multiLineEnabled)) {
                // If multiline is enabled. Escape by ')'
                if ($this->multiLineEnabled) {
                    // Multiline ends...
                    if ($line[0] === ')') {
                        if ($this->syntaxHighLight) {
                            $this->regHighLight('operator', $lineP, strlen($line) - 1);
                        }
                        // Disable multiline
                        $this->multiLineEnabled = 0;
                        $theValue = implode($this->multiLineValue, LF);
                        if (strpos($this->multiLineObject, '.') !== false) {
                            // Set the value deeper.
                            $this->setVal($this->multiLineObject, $setup, array($theValue));
                        } else {
                            // Set value regularly
                            $setup[$this->multiLineObject] = $theValue;
                            if ($this->lastComment && $this->regComments) {
                                $setup[$this->multiLineObject . '..'] .= $this->lastComment;
                            }
                            if ($this->regLinenumbers) {
                                $setup[$this->multiLineObject . '.ln..'][] = $this->lineNumberOffset + $this->rawP - 1;
                            }
                        }
                    } else {
                        if ($this->syntaxHighLight) {
                            $this->regHighLight('value', $lineP);
                        }
                        $this->multiLineValue[] = $this->raw[$this->rawP - 1];
                    }
                } elseif ($this->inBrace === 0 && $line[0] === '[') {
                    // Beginning of condition (only on level zero compared to brace-levels
                    if ($this->syntaxHighLight) {
                        $this->regHighLight('condition', $lineP);
                    }
                    return $line;
                } else {
                    // Return if GLOBAL condition is set - no matter what.
                    if ($line[0] === '[' && stripos($line, '[GLOBAL]') !== false) {
                        if ($this->syntaxHighLight) {
                            $this->regHighLight('condition', $lineP);
                        }
                        $this->error('Line ' . ($this->lineNumberOffset + $this->rawP - 1) . ': On return to [GLOBAL] scope, the script was short of ' . $this->inBrace . ' end brace(s)', 1);
                        $this->inBrace = 0;
                        return $line;
                    } elseif ($line[0] !== '}' && $line[0] !== '#' && $line[0] !== '/') {
                        // If not brace-end or comment
                        // Find object name string until we meet an operator
                        $varL = strcspn($line, TAB . ' {=<>(');
                        // check for special ":=" operator
                        if ($varL > 0 && substr($line, $varL-1, 2) === ':=') {
                            --$varL;
                        }
                        // also remove tabs after the object string name
                        $objStrName = substr($line, 0, $varL);
                        if ($this->syntaxHighLight) {
                            $this->regHighLight('objstr', $lineP, strlen(substr($line, $varL)));
                        }
                        if ($objStrName !== '') {
                            $r = array();
                            if ($this->strict && preg_match('/[^[:alnum:]_\\\\\\.:-]/i', $objStrName, $r)) {
                                $this->error('Line ' . ($this->lineNumberOffset + $this->rawP - 1) . ': Object Name String, "' . htmlspecialchars($objStrName) . '" contains invalid character "' . $r[0] . '". Must be alphanumeric or one of: "_:-\\."');
                            } else {
                                $line = ltrim(substr($line, $varL));
                                if ($this->syntaxHighLight) {
                                    $this->regHighLight('objstr_postspace', $lineP, strlen($line));
                                    if ($line !== '') {
                                        $this->regHighLight('operator', $lineP, strlen($line) - 1);
                                        $this->regHighLight('operator_postspace', $lineP, strlen(ltrim(substr($line, 1))));
                                    }
                                }
                                if ($line === '') {
                                    $this->error('Line ' . ($this->lineNumberOffset + $this->rawP - 1) . ': Object Name String, "' . htmlspecialchars($objStrName) . '" was not followed by any operator, =<>({');
                                } else {
                                    // Checking for special TSparser properties (to change TS values at parsetime)
                                    $match = array();
                                    if ($line[0] === ':' && preg_match('/^:=\\s*([[:alpha:]]+)\\s*\\((.*)\\).*/', $line, $match)) {
                                        $tsFunc = $match[1];
                                        $tsFuncArg = $match[2];
                                        list($currentValue) = $this->getVal($objStrName, $setup);
                                        $tsFuncArg = str_replace(array('\\\\', '\\n', '\\t'), array('\\', LF, TAB), $tsFuncArg);
                                        $newValue = $this->executeValueModifier($tsFunc, $tsFuncArg, $currentValue);
                                        if (isset($newValue)) {
                                            $line = '= ' . $newValue;
                                        }
                                    }
                                    switch ($line[0]) {
                                        case '=':
                                            if ($this->syntaxHighLight) {
                                                $this->regHighLight('value', $lineP, strlen(ltrim(substr($line, 1))) - strlen(trim(substr($line, 1))));
                                            }
                                            if (strpos($objStrName, '.') !== false) {
                                                $value = array();
                                                $value[0] = trim(substr($line, 1));
                                                $this->setVal($objStrName, $setup, $value);
                                            } else {
                                                $setup[$objStrName] = trim(substr($line, 1));
                                                if ($this->lastComment && $this->regComments) {
                                                    // Setting comment..
                                                    $setup[$objStrName . '..'] .= $this->lastComment;
                                                }
                                                if ($this->regLinenumbers) {
                                                    $setup[$objStrName . '.ln..'][] = $this->lineNumberOffset + $this->rawP - 1;
                                                }
                                            }
                                            break;
                                        case '{':
                                            $this->inBrace++;
                                            if (strpos($objStrName, '.') !== false) {
                                                $exitSig = $this->rollParseSub($objStrName, $setup);
                                                if ($exitSig) {
                                                    return $exitSig;
                                                }
                                            } else {
                                                if (!isset($setup[($objStrName . '.')])) {
                                                    $setup[$objStrName . '.'] = array();
                                                }
                                                $exitSig = $this->parseSub($setup[$objStrName . '.']);
                                                if ($exitSig) {
                                                    return $exitSig;
                                                }
                                            }
                                            break;
                                        case '(':
                                            $this->multiLineObject = $objStrName;
                                            $this->multiLineEnabled = 1;
                                            $this->multiLineValue = array();
                                            break;
                                        case '<':
                                            if ($this->syntaxHighLight) {
                                                $this->regHighLight('value_copy', $lineP, strlen(ltrim(substr($line, 1))) - strlen(trim(substr($line, 1))));
                                            }
                                            $theVal = trim(substr($line, 1));
                                            if ($theVal[0] === '.') {
                                                $res = $this->getVal(substr($theVal, 1), $setup);
                                            } else {
                                                $res = $this->getVal($theVal, $this->setup);
                                            }
                                            $this->setVal($objStrName, $setup, unserialize(serialize($res)), 1);
                                            // unserialize(serialize(...)) may look stupid but is needed because of some reference issues. See Kaspers reply to "[TYPO3-core] good question" from December 15 2005.
                                            break;
                                        case '>':
                                            if ($this->syntaxHighLight) {
                                                $this->regHighLight('value_unset', $lineP, strlen(ltrim(substr($line, 1))) - strlen(trim(substr($line, 1))));
                                            }
                                            $this->setVal($objStrName, $setup, 'UNSET');
                                            break;
                                        default:
                                            $this->error('Line ' . ($this->lineNumberOffset + $this->rawP - 1) . ': Object Name String, "' . htmlspecialchars($objStrName) . '" was not followed by any operator, =<>({');
                                    }
                                }
                            }
                            $this->lastComment = '';
                        }
                    } elseif ($line[0] === '}') {
                        $this->inBrace--;
                        $this->lastComment = '';
                        if ($this->syntaxHighLight) {
                            $this->regHighLight('operator', $lineP, strlen($line) - 1);
                        }
                        if ($this->inBrace < 0) {
                            $this->error('Line ' . ($this->lineNumberOffset + $this->rawP - 1) . ': An end brace is in excess.', 1);
                            $this->inBrace = 0;
                        } else {
                            break;
                        }
                    } else {
                        if ($this->syntaxHighLight) {
                            $this->regHighLight('comment', $lineP);
                        }
                        // Comment. The comments are concatenated in this temporary string:
                        if ($this->regComments) {
                            $this->lastComment .= rtrim($line) . LF;
                        }
                    }
                    if (StringUtility::beginsWith($line, '### ERROR')) {
                        $this->error(substr($line, 11));
                    }
                }
            }
            // Unset comment
            if ($this->commentSet) {
                if ($this->syntaxHighLight) {
                    $this->regHighLight('comment', $lineP);
                }
                if (strpos($line, '*/') === 0) {
                    $this->commentSet = 0;
                }
            }
        }
        return null;
    }

    protected function executeValueModifier($modifierName, $modifierArgument = null, $currentValue = null)
    {
        $newValue = null;
        switch ($modifierName) {
            case 'prependString':
                $newValue = $modifierArgument . $currentValue;
                break;
            case 'appendString':
                $newValue = $currentValue . $modifierArgument;
                break;
            case 'removeString':
                $newValue = str_replace($modifierArgument, '', $currentValue);
                break;
            case 'replaceString':
                list($fromStr, $toStr) = explode('|', $modifierArgument, 2);
                $newValue = str_replace($fromStr, $toStr, $currentValue);
                break;
            case 'addToList':
                $newValue = ((string)$currentValue !== '' ? $currentValue . ',' : '') . $modifierArgument;
                break;
            case 'removeFromList':
                $existingElements = GeneralUtility::trimExplode(',', $currentValue);
                $removeElements = GeneralUtility::trimExplode(',', $modifierArgument);
                if (!empty($removeElements)) {
                    $newValue = implode(',', array_diff($existingElements, $removeElements));
                }
                break;
            case 'uniqueList':
                $elements = GeneralUtility::trimExplode(',', $currentValue);
                $newValue = implode(',', array_unique($elements));
                break;
            case 'reverseList':
                $elements = GeneralUtility::trimExplode(',', $currentValue);
                $newValue = implode(',', array_reverse($elements));
                break;
            case 'sortList':
                $elements = GeneralUtility::trimExplode(',', $currentValue);
                $arguments = GeneralUtility::trimExplode(',', $modifierArgument);
                $arguments = array_map('strtolower', $arguments);
                $sort_flags = SORT_REGULAR;
                if (in_array('numeric', $arguments)) {
                    $sort_flags = SORT_NUMERIC;
                    // If the sorting modifier "numeric" is given, all values
                    // are checked and an exception is thrown if a non-numeric value is given
                    // otherwise there is a different behaviour between PHP7 and PHP 5.x
                    // See also the warning on http://us.php.net/manual/en/function.sort.php
                    foreach ($elements as $element) {
                        if (!is_numeric($element)) {
                            throw new \InvalidArgumentException('The list "' . $currentValue . '" should be sorted numerically but contains a non-numeric value', 1438191758);
                        }
                    }
                }
                sort($elements, $sort_flags);
                if (in_array('descending', $arguments)) {
                    $elements = array_reverse($elements);
                }
                $newValue = implode(',', $elements);
                break;
            default:
                if (isset($GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['t3lib/class.t3lib_tsparser.php']['preParseFunc'][$modifierName])) {
                    $hookMethod = $GLOBALS['TYPO3_CONF_VARS']['SC_OPTIONS']['t3lib/class.t3lib_tsparser.php']['preParseFunc'][$modifierName];
                    $params = array('currentValue' => $currentValue, 'functionArgument' => $modifierArgument);
                    $fakeThis = false;
                    $newValue = GeneralUtility::callUserFunction($hookMethod, $params, $fakeThis);
                } else {
                    GeneralUtility::sysLog(
                        'Missing function definition for ' . $modifierName . ' on TypoScript',
                        'core',
                        GeneralUtility::SYSLOG_SEVERITY_WARNING
                    );
                }
        }
        return $newValue;
    }

    public function rollParseSub($string, array &$setup)
    {
        if ((string)$string === '') {
            return '';
        }

        list($key, $remainingKey) = $this->parseNextKeySegment($string);
        $key .= '.';
        if (!isset($setup[$key])) {
            $setup[$key] = array();
        }
        $exitSig = $remainingKey === ''
            ? $this->parseSub($setup[$key])
            : $this->rollParseSub($remainingKey, $setup[$key]);
        return $exitSig ?: '';
    }

    public function getVal($string, $setup)
    {
        if ((string)$string === '') {
            return array();
        }

        list($key, $remainingKey) = $this->parseNextKeySegment($string);
        $subKey = $key . '.';
        if ($remainingKey === '') {
            $retArr = array();
            if (isset($setup[$key])) {
                $retArr[0] = $setup[$key];
            }
            if (isset($setup[$subKey])) {
                $retArr[1] = $setup[$subKey];
            }
            return $retArr;
        } else {
            if ($setup[$subKey]) {
                return $this->getVal($remainingKey, $setup[$subKey]);
            }
        }
        return array();
    }

    public function setVal($string, array &$setup, $value, $wipeOut = false)
    {
        if ((string)$string === '') {
            return;
        }

        list($key, $remainingKey) = $this->parseNextKeySegment($string);
        $subKey = $key . '.';
        if ($remainingKey === '') {
            if ($value === 'UNSET') {
                unset($setup[$key]);
                unset($setup[$subKey]);
                if ($this->regLinenumbers) {
                    $setup[$key . '.ln..'][] = ($this->lineNumberOffset + $this->rawP - 1) . '>';
                }
            } else {
                $lnRegisDone = 0;
                if ($wipeOut && $this->strict) {
                    unset($setup[$key]);
                    unset($setup[$subKey]);
                    if ($this->regLinenumbers) {
                        $setup[$key . '.ln..'][] = ($this->lineNumberOffset + $this->rawP - 1) . '<';
                        $lnRegisDone = 1;
                    }
                }
                if (isset($value[0])) {
                    $setup[$key] = $value[0];
                }
                if (isset($value[1])) {
                    $setup[$subKey] = $value[1];
                }
                if ($this->lastComment && $this->regComments) {
                    $setup[$key . '..'] .= $this->lastComment;
                }
                if ($this->regLinenumbers && !$lnRegisDone) {
                    $setup[$key . '.ln..'][] = $this->lineNumberOffset + $this->rawP - 1;
                }
            }
        } else {
            if (!isset($setup[$subKey])) {
                $setup[$subKey] = array();
            }
            $this->setVal($remainingKey, $setup[$subKey], $value);
        }
    }

    protected function parseNextKeySegment($key)
    {
        // if no dot is in the key, nothing to do
        $dotPosition = strpos($key, '.');
        if ($dotPosition === false) {
            return array($key, '');
        }

        if (strpos($key, '\\') !== false) {
            // backslashes are in the key, so we do further parsing

            while ($dotPosition !== false) {
                if ($dotPosition > 0 && $key[$dotPosition - 1] !== '\\' || $dotPosition > 1 && $key[$dotPosition - 2] === '\\') {
                    break;
                }
                // escaped dot found, continue
                $dotPosition = strpos($key, '.', $dotPosition + 1);
            }

            if ($dotPosition === false) {
                // no regular dot found
                $keySegment = $key;
                $remainingKey = '';
            } else {
                if ($dotPosition > 1 && $key[$dotPosition - 2] === '\\' && $key[$dotPosition - 1] === '\\') {
                    $keySegment = substr($key, 0, $dotPosition - 1);
                } else {
                    $keySegment = substr($key, 0, $dotPosition);
                }
                $remainingKey = substr($key, $dotPosition + 1);
            }

            // fix key segment by removing escape sequences
            $keySegment = str_replace('\\.', '.', $keySegment);
        } else {
            // no backslash in the key, we're fine off
            list($keySegment, $remainingKey) = explode('.', $key, 2);
        }
        return array($keySegment, $remainingKey);
    }

    public function error($err, $num = 2)
    {
        $tt = $this->getTimeTracker();
        if ($tt !== null) {
            $tt->setTSlogMessage($err, $num);
        }
        $this->errors[] = array($err, $num, $this->rawP - 1, $this->lineNumberOffset);
    }

    public static function checkIncludeLines($string, $cycle_counter = 1, $returnFiles = false, $parentFilenameOrPath = '')
    {
        $includedFiles = array();
        if ($cycle_counter > 100) {
            GeneralUtility::sysLog('It appears like TypoScript code is looping over itself. Check your templates for "&lt;INCLUDE_TYPOSCRIPT: ..." tags', 'core', GeneralUtility::SYSLOG_SEVERITY_WARNING);
            if ($returnFiles) {
                return array(
                    'typoscript' => '',
                    'files' => $includedFiles
                );
            }
            return '
###
### ERROR: Recursion!
###
';
        }

        // If no tags found, no need to do slower preg_split
        if (strpos($string, '<INCLUDE_TYPOSCRIPT:') !== false) {
            $splitRegEx = '/\r?\n\s*<INCLUDE_TYPOSCRIPT:\s*(?i)source\s*=\s*"((?i)file|dir):\s*([^"]*)"(.*)>[\ \t]*/';
            $parts = preg_split($splitRegEx, LF . $string . LF, -1, PREG_SPLIT_DELIM_CAPTURE);
            // First text part goes through
            $newString = $parts[0] . LF;
            $partCount = count($parts);
            for ($i = 1; $i + 3 < $partCount; $i += 4) {
                // $parts[$i] contains 'FILE' or 'DIR'
                // $parts[$i+1] contains relative file or directory path to be included
                // $parts[$i+2] optional properties of the INCLUDE statement
                // $parts[$i+3] next part of the typoscript string (part in between include-tags)
                $includeType = $parts[$i];
                $filename = $parts[$i + 1];
                $originalFilename = $filename;
                $optionalProperties = $parts[$i + 2];
                $tsContentsTillNextInclude = $parts[$i + 3];

                // Check condition
                $matches = preg_split('#(?i)condition\\s*=\\s*"((?:\\\\\\\\|\\\\"|[^\\"])*)"(\\s*|>)#', $optionalProperties, 2, PREG_SPLIT_DELIM_CAPTURE);
                // If there was a condition
                if (count($matches) > 1) {
                    // Unescape the condition
                    $condition = trim(stripslashes($matches[1]));
                    // If necessary put condition in square brackets
                    if ($condition[0] !== '[') {
                        $condition = '[' . $condition . ']';
                    }
                    $conditionMatcher = GeneralUtility::makeInstance(ConditionMatcher::class);
                    // If it didn't match then proceed to the next include
                    if (!$conditionMatcher->match($condition)) {
                        continue;
                    }
                }

                // Resolve a possible relative paths if a parent file is given
                if ($parentFilenameOrPath !== '' && $filename[0] === '.') {
                    $filename = PathUtility::getAbsolutePathOfRelativeReferencedFileOrPath($parentFilenameOrPath, $filename);
                }

                // There must be a line-break char after - not sure why this check is necessary, kept it for being 100% backwards compatible
                // An empty string is also ok (means that the next line is also a valid include_typoscript tag)
                if (!preg_match('/(^\\s*\\r?\\n|^$)/', $tsContentsTillNextInclude)) {
                    $newString .= self::typoscriptIncludeError('Invalid characters after <INCLUDE_TYPOSCRIPT: source="' . $includeType . ':' . $filename . '">-tag (rest of line must be empty).');
                } elseif (strpos('..', $filename) !== false) {
                    $newString .= self::typoscriptIncludeError('Invalid filepath "' . $filename . '" (containing "..").');
                } else {
                    switch (strtolower($includeType)) {
                        case 'file':
                            self::includeFile($originalFilename, $cycle_counter, $returnFiles, $newString, $includedFiles, $optionalProperties, $parentFilenameOrPath);
                            break;
                        case 'dir':
                            self::includeDirectory($originalFilename, $cycle_counter, $returnFiles, $newString, $includedFiles, $optionalProperties, $parentFilenameOrPath);
                            break;
                        default:
                            $newString .= self::typoscriptIncludeError('No valid option for INCLUDE_TYPOSCRIPT source property (valid options are FILE or DIR)');
                    }
                }
                // Prepend next normal (not file) part to output string
                $newString .= $tsContentsTillNextInclude . LF;

                // load default TypoScript for content rendering templates like
                // css_styled_content if those have been included through f.e.
                // <INCLUDE_TYPOSCRIPT: source="FILE:EXT:css_styled_content/static/setup.txt">
                $filePointer = strtolower($filename);
                if (StringUtility::beginsWith($filePointer, 'ext:')) {
                    $filePointerPathParts = explode('/', substr($filePointer, 4));

                    // remove file part, determine whether to load setup or constants
                    list($includeType, ) = explode('.', array_pop($filePointerPathParts));

                    if (in_array($includeType, array('setup', 'constants'))) {
                        // adapt extension key to required format (no underscores)
                        $filePointerPathParts[0] = str_replace('_', '', $filePointerPathParts[0]);

                        // load default TypoScript
                        $defaultTypoScriptKey = implode('/', $filePointerPathParts) . '/';
                        if (in_array($defaultTypoScriptKey, $GLOBALS['TYPO3_CONF_VARS']['FE']['contentRenderingTemplates'], true)) {
                            $newString .= $GLOBALS['TYPO3_CONF_VARS']['FE']['defaultTypoScript_' . $includeType . '.']['defaultContentRendering'];
                        }
                    }
                }
            }
            // Add a line break before and after the included code in order to make sure that the parser always has a LF.
            $string = LF . trim($newString) . LF;
        }
        // When all included files should get returned, simply return an compound array containing
        // the TypoScript with all "includes" processed and the files which got included
        if ($returnFiles) {
            return array(
                'typoscript' => $string,
                'files' => $includedFiles
            );
        }
        return $string;
    }

    public static function includeFile($filename, $cycle_counter = 1, $returnFiles = false, &$newString = '', array &$includedFiles = array(), $optionalProperties = '', $parentFilenameOrPath = '')
    {
        // Resolve a possible relative paths if a parent file is given
        if ($parentFilenameOrPath !== '' && $filename[0] === '.') {
            $absfilename = PathUtility::getAbsolutePathOfRelativeReferencedFileOrPath($parentFilenameOrPath, $filename);
        } else {
            $absfilename = $filename;
        }
        $absfilename = GeneralUtility::getFileAbsFileName($absfilename);

        $newString .= LF . '### <INCLUDE_TYPOSCRIPT: source="FILE:' . $filename . '"' . $optionalProperties . '> BEGIN:' . LF;
        if ((string)$filename !== '') {
            // Must exist and must not contain '..' and must be relative
            // Check for allowed files
            if (!GeneralUtility::verifyFilenameAgainstDenyPattern($absfilename)) {
                $newString .= self::typoscriptIncludeError('File "' . $filename . '" was not included since it is not allowed due to fileDenyPattern.');
            } elseif (!@file_exists($absfilename)) {
                $newString .= self::typoscriptIncludeError('File "' . $filename . '" was not found.');
            } else {
                $includedFiles[] = $absfilename;
                // check for includes in included text
                $included_text = self::checkIncludeLines(GeneralUtility::getUrl($absfilename), $cycle_counter + 1, $returnFiles, $absfilename);
                // If the method also has to return all included files, merge currently included
                // files with files included by recursively calling itself
                if ($returnFiles && is_array($included_text)) {
                    $includedFiles = array_merge($includedFiles, $included_text['files']);
                    $included_text = $included_text['typoscript'];
                }
                $newString .= $included_text . LF;
            }
        }
        $newString .= '### <INCLUDE_TYPOSCRIPT: source="FILE:' . $filename . '"' . $optionalProperties . '> END:' . LF . LF;
    }

    protected static function includeDirectory($dirPath, $cycle_counter = 1, $returnFiles = false, &$newString = '', array &$includedFiles = array(), $optionalProperties = '', $parentFilenameOrPath = '')
    {
        // Extract the value of the property extensions="..."
        $matches = preg_split('#(?i)extensions\s*=\s*"([^"]*)"(\s*|>)#', $optionalProperties, 2, PREG_SPLIT_DELIM_CAPTURE);
        if (count($matches) > 1) {
            $includedFileExtensions = $matches[1];
        } else {
            $includedFileExtensions = '';
        }

        // Resolve a possible relative paths if a parent file is given
        if ($parentFilenameOrPath !== '' && $dirPath[0] === '.') {
            $resolvedDirPath = PathUtility::getAbsolutePathOfRelativeReferencedFileOrPath($parentFilenameOrPath, $dirPath);
        } else {
            $resolvedDirPath = $dirPath;
        }
        $absDirPath = GeneralUtility::getFileAbsFileName($resolvedDirPath);
        if ($absDirPath) {
            $absDirPath = rtrim($absDirPath, '/') . '/';
            $newString .= LF . '### <INCLUDE_TYPOSCRIPT: source="DIR:' . $dirPath . '"' . $optionalProperties . '> BEGIN:' . LF;
            // Get alphabetically sorted file index in array
            $fileIndex = GeneralUtility::getAllFilesAndFoldersInPath(array(), $absDirPath, $includedFileExtensions);
            // Prepend file contents to $newString
            $prefixLength = strlen(PATH_site);
            foreach ($fileIndex as $absFileRef) {
                $relFileRef = substr($absFileRef, $prefixLength);
                self::includeFile($relFileRef, $cycle_counter, $returnFiles, $newString, $includedFiles, '', $absDirPath);
            }
            $newString .= '### <INCLUDE_TYPOSCRIPT: source="DIR:' . $dirPath . '"' . $optionalProperties . '> END:' . LF . LF;
        } else {
            $newString .= self::typoscriptIncludeError('The path "' . $resolvedDirPath . '" is invalid.');
        }
    }

    protected static function typoscriptIncludeError($error)
    {
        GeneralUtility::sysLog($error, 'core', GeneralUtility::SYSLOG_SEVERITY_WARNING);
        return "\n###\n### ERROR: " . $error . "\n###\n\n";
    }

    public static function checkIncludeLines_array(array $array)
    {
        foreach ($array as $k => $v) {
            $array[$k] = self::checkIncludeLines($array[$k]);
        }
        return $array;
    }

    public static function extractIncludes($string, $cycle_counter = 1, array $extractedFileNames = array(), $parentFilenameOrPath = '')
    {
        if ($cycle_counter > 10) {
            GeneralUtility::sysLog('It appears like TypoScript code is looping over itself. Check your templates for "&lt;INCLUDE_TYPOSCRIPT: ..." tags', 'core', GeneralUtility::SYSLOG_SEVERITY_WARNING);
            return '
###
### ERROR: Recursion!
###
';
        }
        $expectedEndTag = '';
        $fileContent = array();
        $restContent = array();
        $fileName = null;
        $inIncludePart = false;
        $lines = preg_split("/\r\n|\n|\r/", $string);
        $skipNextLineIfEmpty = false;
        $openingCommentedIncludeStatement = null;
        $optionalProperties = '';
        foreach ($lines as $line) {
            // \TYPO3\CMS\Core\TypoScript\Parser\TypoScriptParser::checkIncludeLines inserts
            // an additional empty line, remove this again
            if ($skipNextLineIfEmpty) {
                if (trim($line) === '') {
                    continue;
                }
                $skipNextLineIfEmpty = false;
            }

            // Outside commented include statements
            if (!$inIncludePart) {
                // Search for beginning commented include statements
                if (preg_match('/###\\s*<INCLUDE_TYPOSCRIPT:\\s*source\\s*=\\s*"\\s*((?i)file|dir)\\s*:\\s*([^"]*)"(.*)>\\s*BEGIN/i', $line, $matches)) {
                    // Found a commented include statement

                    // Save this line in case there is no ending tag
                    $openingCommentedIncludeStatement = trim($line);
                    $openingCommentedIncludeStatement = preg_replace('/\\s*### Warning: .*###\\s*/', '', $openingCommentedIncludeStatement);

                    // type of match: FILE or DIR
                    $inIncludePart = strtoupper($matches[1]);
                    $fileName = $matches[2];
                    $optionalProperties = $matches[3];

                    $expectedEndTag = '### <INCLUDE_TYPOSCRIPT: source="' . $inIncludePart . ':' . $fileName . '"' . $optionalProperties . '> END';
                    // Strip all whitespace characters to make comparison safer
                    $expectedEndTag = strtolower(preg_replace('/\s/', '', $expectedEndTag));
                } else {
                    // If this is not a beginning commented include statement this line goes into the rest content
                    $restContent[] = $line;
                }
                //if (is_array($matches)) GeneralUtility::devLog('matches', 'TypoScriptParser', 0, $matches);
            } else {
                // Inside commented include statements
                // Search for the matching ending commented include statement
                $strippedLine = preg_replace('/\s/', '', $line);
                if (stripos($strippedLine, $expectedEndTag) !== false) {
                    // Found the matching ending include statement
                    $fileContentString = implode(PHP_EOL, $fileContent);

                    // Write the content to the file

                    // Resolve a possible relative paths if a parent file is given
                    if ($parentFilenameOrPath !== '' && $fileName[0] === '.') {
                        $realFileName = PathUtility::getAbsolutePathOfRelativeReferencedFileOrPath($parentFilenameOrPath, $fileName);
                    } else {
                        $realFileName = $fileName;
                    }
                    $realFileName = GeneralUtility::getFileAbsFileName($realFileName);

                    if ($inIncludePart === 'FILE') {
                        // Some file checks
                        if (!GeneralUtility::verifyFilenameAgainstDenyPattern($realFileName)) {
                            throw new \UnexpectedValueException(sprintf('File "%s" was not included since it is not allowed due to fileDenyPattern.', $fileName), 1382651858);
                        }
                        if (empty($realFileName)) {
                            throw new \UnexpectedValueException(sprintf('"%s" is not a valid file location.', $fileName), 1294586441);
                        }
                        if (!is_writable($realFileName)) {
                            throw new \RuntimeException(sprintf('"%s" is not writable.', $fileName), 1294586442);
                        }
                        if (in_array($realFileName, $extractedFileNames)) {
                            throw new \RuntimeException(sprintf('Recursive/multiple inclusion of file "%s"', $realFileName), 1294586443);
                        }
                        $extractedFileNames[] = $realFileName;

                        // Recursive call to detected nested commented include statements
                        $fileContentString = self::extractIncludes($fileContentString, $cycle_counter + 1, $extractedFileNames, $realFileName);

                        // Write the content to the file
                        if (!GeneralUtility::writeFile($realFileName, $fileContentString)) {
                            throw new \RuntimeException(sprintf('Could not write file "%s"', $realFileName), 1294586444);
                        }
                        // Insert reference to the file in the rest content
                        $restContent[] = '<INCLUDE_TYPOSCRIPT: source="FILE:' . $fileName . '"' . $optionalProperties . '>';
                    } else {
                        // must be DIR

                        // Some file checks
                        if (empty($realFileName)) {
                            throw new \UnexpectedValueException(sprintf('"%s" is not a valid location.', $fileName), 1366493602);
                        }
                        if (!is_dir($realFileName)) {
                            throw new \RuntimeException(sprintf('"%s" is not a directory.', $fileName), 1366493603);
                        }
                        if (in_array($realFileName, $extractedFileNames)) {
                            throw new \RuntimeException(sprintf('Recursive/multiple inclusion of directory "%s"', $realFileName), 1366493604);
                        }
                        $extractedFileNames[] = $realFileName;

                        // Recursive call to detected nested commented include statements
                        self::extractIncludes($fileContentString, $cycle_counter + 1, $extractedFileNames, $realFileName);

                        // just drop content between tags since it should usually just contain individual files from that dir

                        // Insert reference to the dir in the rest content
                        $restContent[] = '<INCLUDE_TYPOSCRIPT: source="DIR:' . $fileName . '"' . $optionalProperties . '>';
                    }

                    // Reset variables (preparing for the next commented include statement)
                    $fileContent = array();
                    $fileName = null;
                    $inIncludePart = false;
                    $openingCommentedIncludeStatement = null;
                    // \TYPO3\CMS\Core\TypoScript\Parser\TypoScriptParser::checkIncludeLines inserts
                    // an additional empty line, remove this again
                    $skipNextLineIfEmpty = true;
                } else {
                    // If this is not an ending commented include statement this line goes into the file content
                    $fileContent[] = $line;
                }
            }
        }
        // If we're still inside commented include statements copy the lines back to the rest content
        if ($inIncludePart) {
            $restContent[] = $openingCommentedIncludeStatement . ' ### Warning: Corresponding end line missing! ###';
            $restContent = array_merge($restContent, $fileContent);
        }
        $restContentString = implode(PHP_EOL, $restContent);
        return $restContentString;
    }

    public static function extractIncludes_array(array $array)
    {
        foreach ($array as $k => $v) {
            $array[$k] = self::extractIncludes($array[$k]);
        }
        return $array;
    }

    /**********************************
     *
     * Syntax highlighting
     *
     *********************************/
    public function doSyntaxHighlight($string, $lineNum = '', $highlightBlockMode = false)
    {
        $this->syntaxHighLight = 1;
        $this->highLightData = array();
        $this->errors = array();
        // This is done in order to prevent empty <span>..</span> sections around CR content. Should not do anything but help lessen the amount of HTML code.
        $string = str_replace(CR, '', $string);
        $this->parse($string);
        return $this->syntaxHighlight_print($lineNum, $highlightBlockMode);
    }

    public function regHighLight($code, $pointer, $strlen = -1)
    {
        if ($strlen === -1) {
            $this->highLightData[$pointer] = array(array($code, 0));
        } else {
            $this->highLightData[$pointer][] = array($code, $strlen);
        }
        $this->highLightData_bracelevel[$pointer] = $this->inBrace;
    }

    public function syntaxHighlight_print($lineNumDat, $highlightBlockMode)
    {
        // Registers all error messages in relation to their linenumber
        $errA = array();
        foreach ($this->errors as $err) {
            $errA[$err[2]][] = $err[0];
        }
        // Generates the syntax highlighted output:
        $lines = array();
        foreach ($this->raw as $rawP => $value) {
            $start = 0;
            $strlen = strlen($value);
            $lineC = '';
            if (is_array($this->highLightData[$rawP])) {
                foreach ($this->highLightData[$rawP] as $set) {
                    $len = $strlen - $start - $set[1];
                    if ($len > 0) {
                        $part = substr($value, $start, $len);
                        $start += $len;
                        $st = $this->highLightStyles[isset($this->highLightStyles[$set[0]]) ? $set[0] : 'default'];
                        if (!$highlightBlockMode || $set[0] !== 'prespace') {
                            $lineC .= $st[0] . htmlspecialchars($part) . $st[1];
                        }
                    } elseif ($len < 0) {
                        debug(array($len, $value, $rawP));
                    }
                }
            } else {
                debug(array($value));
            }
            if (strlen($value) > $start) {
                $lineC .= $this->highLightStyles['ignored'][0] . htmlspecialchars(substr($value, $start)) . $this->highLightStyles['ignored'][1];
            }
            if ($errA[$rawP]) {
                $lineC .= $this->highLightStyles['error'][0] . '<strong> - ERROR:</strong> ' . htmlspecialchars(implode(';', $errA[$rawP])) . $this->highLightStyles['error'][1];
            }
            if ($highlightBlockMode && $this->highLightData_bracelevel[$rawP]) {
                $lineC = str_pad('', $this->highLightData_bracelevel[$rawP] * 2, ' ', STR_PAD_LEFT) . '<span style="' . $this->highLightBlockStyles . ($this->highLightBlockStyles_basecolor ? 'background-color: ' . $this->modifyHTMLColorAll($this->highLightBlockStyles_basecolor, -$this->highLightData_bracelevel[$rawP] * 16) : '') . '">' . ($lineC !== '' ? $lineC : '&nbsp;') . '</span>';
            }
            if (is_array($lineNumDat)) {
                $lineNum = $rawP + $lineNumDat[0];
                if ($this->parentObject instanceof ExtendedTemplateService) {
                    $lineNum = $this->parentObject->ext_lnBreakPointWrap($lineNum, $lineNum);
                }
                $lineC = $this->highLightStyles['linenum'][0] . str_pad($lineNum, 4, ' ', STR_PAD_LEFT) . ':' . $this->highLightStyles['linenum'][1] . ' ' . $lineC;
            }
            $lines[] = $lineC;
        }
        return '<pre class="ts-hl">' . implode(LF, $lines) . '</pre>';
    }

    protected function getTimeTracker()
    {
        return isset($GLOBALS['TT']) ? $GLOBALS['TT'] : null;
    }


    protected function modifyHTMLColor($color, $R, $G, $B)
    {
        // This takes a hex-color (# included!) and adds $R, $G and $B to the HTML-color (format: #xxxxxx) and returns the new color
        $nR = MathUtility::forceIntegerInRange(hexdec(substr($color, 1, 2)) + $R, 0, 255);
        $nG = MathUtility::forceIntegerInRange(hexdec(substr($color, 3, 2)) + $G, 0, 255);
        $nB = MathUtility::forceIntegerInRange(hexdec(substr($color, 5, 2)) + $B, 0, 255);
        return '#' . substr(('0' . dechex($nR)), -2) . substr(('0' . dechex($nG)), -2) . substr(('0' . dechex($nB)), -2);
    }

    protected function modifyHTMLColorAll($color, $all)
    {
        return $this->modifyHTMLColor($color, $all, $all, $all);
    }
}