josh/jes
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Support loading files that do not end with a EOL sequence

Browse Source 
- do not drop the last line of such files from TextLoader
- add a flag to TextLoader to indicate if the loaded file ended with a
  EOL sequence
This commit is contained in:
Josh Holtrop 2016-07-24 12:22:03 -04:00
parent 89e8dc5de3
commit 7093560a94
3 changed files with 32 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/core/TextLoader.cc
View File

@ -60,16 +60,23 @@ void TextLoader::load_buffer(uint8_t * buffer, size_t size)
if (crlf && (n_lf > 0u))
{
m_line_endings = LINE_ENDING_CRLF;
m_lines = lines[LINE_ENDING_CRLF];
}
else if ((n_cr > 0u) && (n_lf == 0u))
{
m_line_endings = LINE_ENDING_CR;
m_lines = lines[LINE_ENDING_CR];
}
else
{
m_line_endings = LINE_ENDING_LF;
m_lines = lines[LINE_ENDING_LF];
}
m_lines = lines[m_line_endings];
/* Check if there is a line that was not terminated by a EOL sequence at
* the end of the file. */
if (line_start[m_line_endings] < size)
{
m_lines->push_back(LineIndexPair(&buffer[line_start[m_line_endings]], size - line_start[m_line_endings]));
m_eol_at_eof = false;
}
}

2
test/files/no_eol_at_eof.txt Normal file
View File

@ -0,0 +1,2 @@
Line 1
Line 2

27
test/src/test_TextLoader.cc
View File

@ -23,7 +23,7 @@ TEST(TextLoaderTest, loading_empty_file)
auto file = TestSupport::read_file("test/files/empty.txt");
tl.load_buffer(&(*file)[0], file->size());
EXPECT_EQ(0u, tl.num_lines());
EXPECT_EQ(true, tl.get_eol_at_eof());
EXPECT_TRUE(tl.get_eol_at_eof());
}
TEST(TextLoaderTest, detects_lf_line_endings)
@ -32,12 +32,12 @@ TEST(TextLoaderTest, detects_lf_line_endings)
auto file = TestSupport::read_file("test/files/line_endings/lf_format.txt");
tl.load_buffer(&(*file)[0], file->size());
EXPECT_EQ(TextLoader::LINE_ENDING_LF, tl.get_line_endings());
EXPECT_EQ(2u, tl.num_lines());
ASSERT_EQ(2u, tl.num_lines());
auto it = tl.begin();
EXPECT_EQ("Hello.", line_to_string(it));
it++;
EXPECT_EQ("This file is in LF line ending format.", line_to_string(it));
EXPECT_EQ(true, tl.get_eol_at_eof());
EXPECT_TRUE(tl.get_eol_at_eof());
}
TEST(TextLoaderTest, detects_cr_line_endings)
@ -46,12 +46,12 @@ TEST(TextLoaderTest, detects_cr_line_endings)
auto file = TestSupport::read_file("test/files/line_endings/cr_format.txt");
tl.load_buffer(&(*file)[0], file->size());
EXPECT_EQ(TextLoader::LINE_ENDING_CR, tl.get_line_endings());
EXPECT_EQ(2u, tl.num_lines());
ASSERT_EQ(2u, tl.num_lines());
auto it = tl.begin();
EXPECT_EQ("Hello.", line_to_string(it));
it++;
EXPECT_EQ("This file is in CR line ending format.", line_to_string(it));
EXPECT_EQ(true, tl.get_eol_at_eof());
EXPECT_TRUE(tl.get_eol_at_eof());
}
TEST(TextLoaderTest, detects_crlf_line_endings)
@ -60,10 +60,23 @@ TEST(TextLoaderTest, detects_crlf_line_endings)
auto file = TestSupport::read_file("test/files/line_endings/crlf_format.txt");
tl.load_buffer(&(*file)[0], file->size());
EXPECT_EQ(TextLoader::LINE_ENDING_CRLF, tl.get_line_endings());
EXPECT_EQ(2u, tl.num_lines());
ASSERT_EQ(2u, tl.num_lines());
auto it = tl.begin();
EXPECT_EQ("Hello.", line_to_string(it));
it++;
EXPECT_EQ("This file is in CRLF line ending format.", line_to_string(it));
EXPECT_EQ(true, tl.get_eol_at_eof());
EXPECT_TRUE(tl.get_eol_at_eof());
}
TEST(TextLoaderTest, properly_reads_files_that_do_not_end_in_a_eol_sequence)
{
TextLoader tl;
auto file = TestSupport::read_file("test/files/no_eol_at_eof.txt");
tl.load_buffer(&(*file)[0], file->size());
ASSERT_EQ(2u, tl.num_lines());
auto it = tl.begin();
EXPECT_EQ("Line 1", line_to_string(it));
it++;
EXPECT_EQ("Line 2", line_to_string(it));
EXPECT_FALSE(tl.get_eol_at_eof());
}