Skip to content

Commit

Permalink
IO: bug fixed in string buffer
Browse files Browse the repository at this point in the history
  • Loading branch information
@remorhaz
remorhaz committed Apr 28, 2018
1 parent 41fbc02 commit b41a2ec
Show file tree
Hide file tree
Showing 5 changed files with 239 additions and 11 deletions.
2 changes: 1 addition & 1 deletion src/IO/StringBuffer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ public function getTokenPosition(): TokenPosition

public function getTokenAsString(): string
{
return substr($this->data, $this->startOffset, $this->previewOffset);
return substr($this->data, $this->startOffset, $this->previewOffset - $this->startOffset);
}

public function getTokenAsArray(): array
Expand Down
31 changes: 29 additions & 2 deletions src/Unicode/CharBuffer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

use Remorhaz\UniLex\Exception;
use Remorhaz\UniLex\IO\CharBufferInterface;
use Remorhaz\UniLex\IO\TokenExtractInterface;
use Remorhaz\UniLex\Lexer\Token;
use Remorhaz\UniLex\Lexer\TokenFactoryInterface;
use Remorhaz\UniLex\Lexer\TokenMatcherInterface;
Expand All @@ -13,7 +14,7 @@
use Remorhaz\UniLex\Unicode\Grammar\TokenType;
use Remorhaz\UniLex\Unicode\Grammar\Utf8TokenMatcher;

class CharBuffer implements CharBufferInterface
class CharBuffer implements CharBufferInterface, TokenExtractInterface
{

private $source;
Expand All @@ -28,6 +29,8 @@ class CharBuffer implements CharBufferInterface

private $previewOffset = 0;

private $buffer = [];

public function __construct(CharBufferInterface $source)
{
$this->source = $source;
Expand Down Expand Up @@ -81,7 +84,9 @@ private function getMatchedChar(): int
if ($token->getType() != TokenType::SYMBOL) {
throw new Exception("Invalid Unicode char token");
}
return $token->getAttribute(TokenAttribute::UNICODE_CHAR);
$char = $token->getAttribute(TokenAttribute::UNICODE_CHAR);
$this->buffer[] = $char;
return $char;
}

/**
Expand Down Expand Up @@ -109,12 +114,14 @@ public function finishToken(Token $token): void
$token->setAttribute(TokenAttribute::UNICODE_CHAR_OFFSET_START, $this->startOffset);
$token->setAttribute(TokenAttribute::UNICODE_CHAR_OFFSET_FINISH, $this->previewOffset);
$this->startOffset = $this->previewOffset;
$this->buffer = [];
}

public function resetToken(): void
{
$this->previewOffset = $this->startOffset;
$this->source->resetToken();
$this->buffer = [];
unset($this->char);
}

Expand All @@ -127,6 +134,26 @@ public function getTokenPosition(): TokenPosition
return new TokenPosition($this->startOffset, $this->previewOffset);
}

/**
* @return string
* @throws Exception
*/
public function getTokenAsString(): string
{
if ($this->source instanceof TokenExtractInterface) {
return $this->source->getTokenAsString();
}
throw new Exception("Source buffer doesn't support extracting strings");
}

/**
* @return array
*/
public function getTokenAsArray(): array
{
return $this->buffer;
}

private function getMatcher(): TokenMatcherInterface
{
if (!isset($this->matcher)) {
Expand Down
56 changes: 52 additions & 4 deletions tests/CharBufferTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public function testGetTokenPosition_NextSymbolAndFinishTokenCalled_ReturnsMatch
self::assertSame(1, $position->getFinishOffset());
}

public function testGetTokenAsArray_EmptyPosition_ReturnsEmptyArray(): void
public function testGetTokenAsArray_NextSymbolNotCalled_ReturnsEmptyArray(): void
{
$actualValue = (new CharBuffer(0x61))->getTokenAsArray();
self::assertSame([], $actualValue);
Expand All @@ -133,7 +133,7 @@ public function testGetTokenAsArray_EmptyPosition_ReturnsEmptyArray(): void
/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsArray_NotEmptyPosition_ReturnsMatchingArray(): void
public function testGetTokenAsArray_NextSymbolCalledTwice_ReturnsMatchingArray(): void
{
$buffer = new CharBuffer(0x61, 0x62);
$buffer->nextSymbol();
Expand All @@ -145,7 +145,31 @@ public function testGetTokenAsArray_NotEmptyPosition_ReturnsMatchingArray(): voi
/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsString_EmptyPosition_ReturnsEmptyString(): void
public function testGetTokenAsArray_NextSymbolAndResetTokenCalled_ReturnsEmptyArray(): void
{
$buffer = new CharBuffer(0x61, 0x62);
$buffer->nextSymbol();
$buffer->resetToken();
$actualValue = $buffer->getTokenAsArray();
self::assertSame([], $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsArray_NextSymbolAndFinishTokenCalled_ReturnsEmptyArray(): void
{
$buffer = new CharBuffer(0x61, 0x62);
$buffer->nextSymbol();
$buffer->finishToken(new Token(0, false));
$actualValue = $buffer->getTokenAsArray();
self::assertSame([], $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsString_NextSymbolNotCalled_ReturnsEmptyString(): void
{
$actualValue = (new CharBuffer(0x61))->getTokenAsString();
self::assertSame('', $actualValue);
Expand All @@ -154,7 +178,7 @@ public function testGetTokenAsString_EmptyPosition_ReturnsEmptyString(): void
/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsString_NotEmptyPosition_ReturnsMatchingArray(): void
public function testGetTokenAsString_NextSymbolCalledTwice_ReturnsMatchingString(): void
{
$buffer = new CharBuffer(0x61, 0x62);
$buffer->nextSymbol();
Expand All @@ -163,6 +187,30 @@ public function testGetTokenAsString_NotEmptyPosition_ReturnsMatchingArray(): vo
self::assertSame('ab', $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsString_NextSymbolAndResetTokenCalled_ReturnsEmptyString(): void
{
$buffer = new CharBuffer(0x61, 0x62);
$buffer->nextSymbol();
$buffer->resetToken();
$actualValue = $buffer->getTokenAsString();
self::assertSame('', $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsString_NextSymbolAndFinishTokenCalled_ReturnsEmptyString(): void
{
$buffer = new CharBuffer(0x61, 0x62);
$buffer->nextSymbol();
$buffer->finishToken(new Token(0, false));
$actualValue = $buffer->getTokenAsString();
self::assertSame('', $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
* @expectedException \Remorhaz\UniLex\Exception
Expand Down
56 changes: 52 additions & 4 deletions tests/StringBufferTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public function testGetTokenPosition_NextSymbolAndFinishTokenCalled_ReturnsMatch
self::assertSame(1, $position->getFinishOffset());
}

public function testGetTokenAsArray_EmptyPosition_ReturnsEmptyArray(): void
public function testGetTokenAsArray_NextSymbolNotCalled_ReturnsEmptyArray(): void
{
$actualValue = (new StringBuffer('a'))->getTokenAsArray();
self::assertSame([], $actualValue);
Expand All @@ -133,7 +133,7 @@ public function testGetTokenAsArray_EmptyPosition_ReturnsEmptyArray(): void
/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsArray_NotEmptyPosition_ReturnsMatchingArray(): void
public function testGetTokenAsArray_NextSymbolCalledTwice_ReturnsMatchingArray(): void
{
$buffer = new StringBuffer('ab');
$buffer->nextSymbol();
Expand All @@ -142,7 +142,31 @@ public function testGetTokenAsArray_NotEmptyPosition_ReturnsMatchingArray(): voi
self::assertSame([0x61, 0x62], $actualValue);
}

public function testGetTokenAsString_EmptyPosition_ReturnsEmptyString(): void
/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsArray_NextSymbolAndResetTokenCalled_ReturnsEmptyArray(): void
{
$buffer = new StringBuffer('ab');
$buffer->nextSymbol();
$buffer->resetToken();
$actualValue = $buffer->getTokenAsArray();
self::assertSame([], $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsArray_NextSymbolAndFinishTokenCalled_ReturnsEmptyArray(): void
{
$buffer = new StringBuffer('ab');
$buffer->nextSymbol();
$buffer->finishToken(new Token(0, false));
$actualValue = $buffer->getTokenAsArray();
self::assertSame([], $actualValue);
}

public function testGetTokenAsString_NextSymbolNotCalled_ReturnsEmptyString(): void
{
$actualValue = (new StringBuffer('a'))->getTokenAsString();
self::assertSame('', $actualValue);
Expand All @@ -152,12 +176,36 @@ public function testGetTokenAsString_EmptyPosition_ReturnsEmptyString(): void
/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsString_NotEmptyPosition_ReturnsMatchingArray(): void
public function testGetTokenAsString_NextSymbolCalledTwice_ReturnsMatchingString(): void
{
$buffer = new StringBuffer('ab');
$buffer->nextSymbol();
$buffer->nextSymbol();
$actualValue = $buffer->getTokenAsString();
self::assertSame('ab', $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsString_NextSymbolAndResetTokenCalled_ReturnsEmptyString(): void
{
$buffer = new StringBuffer('ab');
$buffer->nextSymbol();
$buffer->resetToken();
$actualValue = $buffer->getTokenAsString();
self::assertSame('', $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsString_NextSymbolAndFinishTokenCalled_ReturnsEmptyString(): void
{
$buffer = new StringBuffer('ab');
$buffer->nextSymbol();
$buffer->finishToken(new Token(0, false));
$actualValue = $buffer->getTokenAsString();
self::assertSame('', $actualValue);
}
}
105 changes: 105 additions & 0 deletions tests/Unicode/CharBufferTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,103 @@ public function testResetToken_NextSymbolCalled_FinishTokenSetsZeroByteOffsetsIn
self::assertSame(0, $token->getAttribute(TokenAttribute::UNICODE_BYTE_OFFSET_FINISH));
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsString_NextSymbolNotCalled_ReturnsEmptyString(): void
{
$source = new StringBuffer('a');
$buffer = new CharBuffer($source);
$actualValue = $buffer->getTokenAsString();
self::assertSame('', $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsString_NextSymbolCalledTwice_ReturnsMatchingString(): void
{
$source = new StringBuffer('ab');
$buffer = new CharBuffer($source);
$buffer->nextSymbol();
$buffer->nextSymbol();
$actualValue = $buffer->getTokenAsString();
self::assertSame('ab', $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsString_NextSymbolAndResetTokenCalled_ReturnsEmptyString(): void
{
$source = new StringBuffer('ab');
$buffer = new CharBuffer($source);
$buffer->nextSymbol();
$buffer->resetToken();
$actualValue = $buffer->getTokenAsString();
self::assertSame('', $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsString_NextSymbolAndFinishTokenCalled_ReturnsEmptyString(): void
{
$source = new StringBuffer('ab');
$buffer = new CharBuffer($source);
$buffer->nextSymbol();
$buffer->finishToken(new Token(0, false));
$actualValue = $buffer->getTokenAsString();
self::assertSame('', $actualValue);
}

public function testGetTokenAsArray_NextSymbolNotCalled_ReturnsEmptyArray(): void
{
$source = new StringBuffer('ab');
$buffer = new CharBuffer($source);
$actualValue = $buffer->getTokenAsArray();
self::assertSame([], $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsArray_NextSymbolCalledTwice_ReturnsMatchingArray(): void
{
$source = new StringBuffer('ab');
$buffer = new CharBuffer($source);
$buffer->nextSymbol();
$buffer->nextSymbol();
$actualValue = $buffer->getTokenAsArray();
self::assertSame([0x61, 0x62], $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsArray_NextSymbolAndResetTokenCalled_ReturnsEmptyArray(): void
{
$source = new StringBuffer('ab');
$buffer = new CharBuffer($source);
$buffer->nextSymbol();
$buffer->resetToken();
$actualValue = $buffer->getTokenAsArray();
self::assertSame([], $actualValue);
}

/**
* @throws \Remorhaz\UniLex\Exception
*/
public function testGetTokenAsArray_NextSymbolAndFinishTokenCalled_ReturnsEmptyArray(): void
{
$source = new StringBuffer('ab');
$buffer = new CharBuffer($source);
$buffer->nextSymbol();
$buffer->finishToken(new Token(0, false));
$actualValue = $buffer->getTokenAsArray();
self::assertSame([], $actualValue);
}

private function createTokenMatcherThatNeverMatches(): TokenMatcherInterface
{
return new class implements TokenMatcherInterface
Expand All @@ -221,6 +318,10 @@ public function match(CharBufferInterface $buffer, TokenFactoryInterface $tokenF
return false;
}

/**
* @return Token
* @throws Exception
*/
public function getToken(): Token
{
throw new Exception("Not implemented");
Expand All @@ -238,6 +339,10 @@ public function createToken(int $tokenId): Token
return new Token(TokenType::INVALID_BYTES, false);
}

/**
* @return Token
* @throws Exception
*/
public function createEoiToken(): Token
{
throw new Exception("Not implemented");
Expand Down

0 comments on commit b41a2ec

Please sign in to comment.