www.aleph1.co.uk Git - yaffs-website/blob - vendor/nikic/php-parser/test/PhpParser/LexerTest.php

   1 <?php declare(strict_types=1);
   2
   3 namespace PhpParser;
   4
   5 use PhpParser\Parser\Tokens;
   6 use PHPUnit\Framework\TestCase;
   7
   8 class LexerTest extends TestCase
   9 {
  10     /* To allow overwriting in parent class */
  11     protected function getLexer(array $options = []) {
  12         return new Lexer($options);
  13     }
  14
  15     /**
  16      * @dataProvider provideTestError
  17      */
  18     public function testError($code, $messages) {
  19         if (defined('HHVM_VERSION')) {
  20             $this->markTestSkipped('HHVM does not throw warnings from token_get_all()');
  21         }
  22
  23         $errorHandler = new ErrorHandler\Collecting();
  24         $lexer = $this->getLexer(['usedAttributes' => [
  25             'comments', 'startLine', 'endLine', 'startFilePos', 'endFilePos'
  26         ]]);
  27         $lexer->startLexing($code, $errorHandler);
  28         $errors = $errorHandler->getErrors();
  29
  30         $this->assertCount(count($messages), $errors);
  31         for ($i = 0; $i < count($messages); $i++) {
  32             $this->assertSame($messages[$i], $errors[$i]->getMessageWithColumnInfo($code));
  33         }
  34     }
  35
  36     public function provideTestError() {
  37         return [
  38             ["<?php /*", ["Unterminated comment from 1:7 to 1:9"]],
  39             ["<?php \1", ["Unexpected character \"\1\" (ASCII 1) from 1:7 to 1:7"]],
  40             ["<?php \0", ["Unexpected null byte from 1:7 to 1:7"]],
  41             // Error with potentially emulated token
  42             ["<?php ?? \0", ["Unexpected null byte from 1:10 to 1:10"]],
  43             ["<?php\n\0\1 foo /* bar", [
  44                 "Unexpected null byte from 2:1 to 2:1",
  45                 "Unexpected character \"\1\" (ASCII 1) from 2:2 to 2:2",
  46                 "Unterminated comment from 2:8 to 2:14"
  47             ]],
  48         ];
  49     }
  50
  51     /**
  52      * @dataProvider provideTestLex
  53      */
  54     public function testLex($code, $options, $tokens) {
  55         $lexer = $this->getLexer($options);
  56         $lexer->startLexing($code);
  57         while ($id = $lexer->getNextToken($value, $startAttributes, $endAttributes)) {
  58             $token = array_shift($tokens);
  59
  60             $this->assertSame($token[0], $id);
  61             $this->assertSame($token[1], $value);
  62             $this->assertEquals($token[2], $startAttributes);
  63             $this->assertEquals($token[3], $endAttributes);
  64         }
  65     }
  66
  67     public function provideTestLex() {
  68         return [
  69             // tests conversion of closing PHP tag and drop of whitespace and opening tags
  70             [
  71                 '<?php tokens ?>plaintext',
  72                 [],
  73                 [
  74                     [
  75                         Tokens::T_STRING, 'tokens',
  76                         ['startLine' => 1], ['endLine' => 1]
  77                     ],
  78                     [
  79                         ord(';'), '?>',
  80                         ['startLine' => 1], ['endLine' => 1]
  81                     ],
  82                     [
  83                         Tokens::T_INLINE_HTML, 'plaintext',
  84                         ['startLine' => 1, 'hasLeadingNewline' => false],
  85                         ['endLine' => 1]
  86                     ],
  87                 ]
  88             ],
  89             // tests line numbers
  90             [
  91                 '<?php' . "\n" . '$ token /** doc' . "\n" . 'comment */ $',
  92                 [],
  93                 [
  94                     [
  95                         ord('$'), '$',
  96                         ['startLine' => 2], ['endLine' => 2]
  97                     ],
  98                     [
  99                         Tokens::T_STRING, 'token',
 100                         ['startLine' => 2], ['endLine' => 2]
 101                     ],
 102                     [
 103                         ord('$'), '$',
 104                         [
 105                             'startLine' => 3,
 106                             'comments' => [
 107                                 new Comment\Doc('/** doc' . "\n" . 'comment */', 2, 14, 5),
 108                             ]
 109                         ],
 110                         ['endLine' => 3]
 111                     ],
 112                 ]
 113             ],
 114             // tests comment extraction
 115             [
 116                 '<?php /* comment */ // comment' . "\n" . '/** docComment 1 *//** docComment 2 */ token',
 117                 [],
 118                 [
 119                     [
 120                         Tokens::T_STRING, 'token',
 121                         [
 122                             'startLine' => 2,
 123                             'comments' => [
 124                                 new Comment('/* comment */', 1, 6, 1),
 125                                 new Comment('// comment' . "\n", 1, 20, 3),
 126                                 new Comment\Doc('/** docComment 1 */', 2, 31, 4),
 127                                 new Comment\Doc('/** docComment 2 */', 2, 50, 5),
 128                             ],
 129                         ],
 130                         ['endLine' => 2]
 131                     ],
 132                 ]
 133             ],
 134             // tests differing start and end line
 135             [
 136                 '<?php "foo' . "\n" . 'bar"',
 137                 [],
 138                 [
 139                     [
 140                         Tokens::T_CONSTANT_ENCAPSED_STRING, '"foo' . "\n" . 'bar"',
 141                         ['startLine' => 1], ['endLine' => 2]
 142                     ],
 143                 ]
 144             ],
 145             // tests exact file offsets
 146             [
 147                 '<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
 148                 ['usedAttributes' => ['startFilePos', 'endFilePos']],
 149                 [
 150                     [
 151                         Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"',
 152                         ['startFilePos' => 6], ['endFilePos' => 8]
 153                     ],
 154                     [
 155                         ord(';'), ';',
 156                         ['startFilePos' => 9], ['endFilePos' => 9]
 157                     ],
 158                     [
 159                         Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"',
 160                         ['startFilePos' => 18], ['endFilePos' => 20]
 161                     ],
 162                     [
 163                         ord(';'), ';',
 164                         ['startFilePos' => 21], ['endFilePos' => 21]
 165                     ],
 166                 ]
 167             ],
 168             // tests token offsets
 169             [
 170                 '<?php "a";' . "\n" . '// foo' . "\n" . '"b";',
 171                 ['usedAttributes' => ['startTokenPos', 'endTokenPos']],
 172                 [
 173                     [
 174                         Tokens::T_CONSTANT_ENCAPSED_STRING, '"a"',
 175                         ['startTokenPos' => 1], ['endTokenPos' => 1]
 176                     ],
 177                     [
 178                         ord(';'), ';',
 179                         ['startTokenPos' => 2], ['endTokenPos' => 2]
 180                     ],
 181                     [
 182                         Tokens::T_CONSTANT_ENCAPSED_STRING, '"b"',
 183                         ['startTokenPos' => 5], ['endTokenPos' => 5]
 184                     ],
 185                     [
 186                         ord(';'), ';',
 187                         ['startTokenPos' => 6], ['endTokenPos' => 6]
 188                     ],
 189                 ]
 190             ],
 191             // tests all attributes being disabled
 192             [
 193                 '<?php /* foo */ $bar;',
 194                 ['usedAttributes' => []],
 195                 [
 196                     [
 197                         Tokens::T_VARIABLE, '$bar',
 198                         [], []
 199                     ],
 200                     [
 201                         ord(';'), ';',
 202                         [], []
 203                     ]
 204                 ]
 205             ],
 206             // tests no tokens
 207             [
 208                 '',
 209                 [],
 210                 []
 211             ],
 212         ];
 213     }
 214
 215     /**
 216      * @dataProvider provideTestHaltCompiler
 217      */
 218     public function testHandleHaltCompiler($code, $remaining) {
 219         $lexer = $this->getLexer();
 220         $lexer->startLexing($code);
 221
 222         while (Tokens::T_HALT_COMPILER !== $lexer->getNextToken());
 223
 224         $this->assertSame($remaining, $lexer->handleHaltCompiler());
 225         $this->assertSame(0, $lexer->getNextToken());
 226     }
 227
 228     public function provideTestHaltCompiler() {
 229         return [
 230             ['<?php ... __halt_compiler();Remaining Text', 'Remaining Text'],
 231             ['<?php ... __halt_compiler ( ) ;Remaining Text', 'Remaining Text'],
 232             ['<?php ... __halt_compiler() ?>Remaining Text', 'Remaining Text'],
 233             //array('<?php ... __halt_compiler();' . "\0", "\0"),
 234             //array('<?php ... __halt_compiler /* */ ( ) ;Remaining Text', 'Remaining Text'),
 235         ];
 236     }
 237
 238     public function testHandleHaltCompilerError() {
 239         $this->expectException(Error::class);
 240         $this->expectExceptionMessage('__HALT_COMPILER must be followed by "();"');
 241         $lexer = $this->getLexer();
 242         $lexer->startLexing('<?php ... __halt_compiler invalid ();');
 243
 244         while (Tokens::T_HALT_COMPILER !== $lexer->getNextToken());
 245         $lexer->handleHaltCompiler();
 246     }
 247
 248     public function testGetTokens() {
 249         $code = '<?php "a";' . "\n" . '// foo' . "\n" . '"b";';
 250         $expectedTokens = [
 251             [T_OPEN_TAG, '<?php ', 1],
 252             [T_CONSTANT_ENCAPSED_STRING, '"a"', 1],
 253             ';',
 254             [T_WHITESPACE, "\n", 1],
 255             [T_COMMENT, '// foo' . "\n", 2],
 256             [T_CONSTANT_ENCAPSED_STRING, '"b"', 3],
 257             ';',
 258         ];
 259
 260         $lexer = $this->getLexer();
 261         $lexer->startLexing($code);
 262         $this->assertSame($expectedTokens, $lexer->getTokens());
 263     }
 264 }