add LineEndings module
This commit is contained in:
parent
cdae9e51ae
commit
382d155e40
@ -11,7 +11,7 @@ Buffer::Buffer()
|
|||||||
m_file_buffer = nullptr;
|
m_file_buffer = nullptr;
|
||||||
m_file_buffer_size = 0u;
|
m_file_buffer_size = 0u;
|
||||||
m_eol_at_eof = true;
|
m_eol_at_eof = true;
|
||||||
m_line_endings = TextLoader::LINE_ENDING_LF;
|
m_line_endings = LineEndings::LF;
|
||||||
}
|
}
|
||||||
|
|
||||||
Buffer::~Buffer()
|
Buffer::~Buffer()
|
||||||
@ -71,26 +71,6 @@ bool Buffer::load_from_file(const char * filename)
|
|||||||
bool Buffer::write_to_file(const char * filename)
|
bool Buffer::write_to_file(const char * filename)
|
||||||
{
|
{
|
||||||
File file;
|
File file;
|
||||||
const char * eol_seq;
|
|
||||||
size_t eol_len;
|
|
||||||
switch (m_line_endings)
|
|
||||||
{
|
|
||||||
case TextLoader::LINE_ENDING_LF:
|
|
||||||
eol_seq = "\n";
|
|
||||||
eol_len = 1u;
|
|
||||||
break;
|
|
||||||
case TextLoader::LINE_ENDING_CR:
|
|
||||||
eol_seq = "\r";
|
|
||||||
eol_len = 1u;
|
|
||||||
break;
|
|
||||||
case TextLoader::LINE_ENDING_CRLF:
|
|
||||||
eol_seq = "\r\n";
|
|
||||||
eol_len = 2u;
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
assert(false);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
if (!file.open(filename, true))
|
if (!file.open(filename, true))
|
||||||
{
|
{
|
||||||
return false;
|
return false;
|
||||||
@ -108,17 +88,19 @@ bool Buffer::write_to_file(const char * filename)
|
|||||||
bytes_written += pd->length;
|
bytes_written += pd->length;
|
||||||
if (pd->eol())
|
if (pd->eol())
|
||||||
{
|
{
|
||||||
if (!file.write((const uint8_t *)eol_seq, eol_len))
|
if (!file.write(LineEndings::spans[m_line_endings].start,
|
||||||
|
LineEndings::spans[m_line_endings].length))
|
||||||
{
|
{
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
bytes_written += eol_len;
|
bytes_written += LineEndings::spans[m_line_endings].length;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (m_eol_at_eof && bytes_written > 0u)
|
if (m_eol_at_eof && bytes_written > 0u)
|
||||||
{
|
{
|
||||||
if (!file.write((const uint8_t *)eol_seq, eol_len))
|
if (!file.write(LineEndings::spans[m_line_endings].start,
|
||||||
|
LineEndings::spans[m_line_endings].length))
|
||||||
{
|
{
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -4,6 +4,7 @@
|
|||||||
#include <list>
|
#include <list>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include "PieceTable.h"
|
#include "PieceTable.h"
|
||||||
|
#include "LineEndings.h"
|
||||||
|
|
||||||
class Buffer
|
class Buffer
|
||||||
{
|
{
|
||||||
@ -19,7 +20,7 @@ protected:
|
|||||||
uint8_t * m_file_buffer;
|
uint8_t * m_file_buffer;
|
||||||
unsigned long m_file_buffer_size;
|
unsigned long m_file_buffer_size;
|
||||||
bool m_eol_at_eof;
|
bool m_eol_at_eof;
|
||||||
uint8_t m_line_endings;
|
LineEndings::Type m_line_endings;
|
||||||
|
|
||||||
void free_file_buffer();
|
void free_file_buffer();
|
||||||
};
|
};
|
||||||
|
11
src/core/LineEndings.cc
Normal file
11
src/core/LineEndings.cc
Normal file
@ -0,0 +1,11 @@
|
|||||||
|
#include "LineEndings.h"
|
||||||
|
|
||||||
|
static uint8_t lf[] = "\n";
|
||||||
|
static uint8_t crlf[] = "\r\n";
|
||||||
|
static uint8_t cr[] = "\r";
|
||||||
|
|
||||||
|
const Span LineEndings::spans[] = {
|
||||||
|
Span(lf, 1u),
|
||||||
|
Span(crlf, 2u),
|
||||||
|
Span(cr, 1u),
|
||||||
|
};
|
20
src/core/LineEndings.h
Normal file
20
src/core/LineEndings.h
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
#ifndef LINEENDINGS_H
|
||||||
|
#define LINEENDINGS_H
|
||||||
|
|
||||||
|
#include "Span.h"
|
||||||
|
|
||||||
|
class LineEndings
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
enum Type
|
||||||
|
{
|
||||||
|
LF,
|
||||||
|
CRLF,
|
||||||
|
CR,
|
||||||
|
COUNT,
|
||||||
|
};
|
||||||
|
|
||||||
|
static const Span spans[COUNT];
|
||||||
|
};
|
||||||
|
|
||||||
|
#endif
|
@ -4,7 +4,7 @@
|
|||||||
/** Create a TextLoader. */
|
/** Create a TextLoader. */
|
||||||
TextLoader::TextLoader()
|
TextLoader::TextLoader()
|
||||||
{
|
{
|
||||||
m_line_endings = LINE_ENDING_LF;
|
m_line_endings = LineEndings::LF;
|
||||||
m_lines = NULL;
|
m_lines = NULL;
|
||||||
m_eol_at_eof = true;
|
m_eol_at_eof = true;
|
||||||
}
|
}
|
||||||
@ -17,12 +17,12 @@ TextLoader::TextLoader()
|
|||||||
*/
|
*/
|
||||||
void TextLoader::load_buffer(uint8_t * buffer, size_t size)
|
void TextLoader::load_buffer(uint8_t * buffer, size_t size)
|
||||||
{
|
{
|
||||||
std::shared_ptr<std::list<Span>> lines[LINE_ENDING_COUNT];
|
std::shared_ptr<std::list<Span>> lines[LineEndings::COUNT];
|
||||||
size_t line_start[LINE_ENDING_COUNT] = {0};
|
size_t line_start[LineEndings::COUNT] = {0};
|
||||||
unsigned int n_cr = 0;
|
unsigned int n_cr = 0;
|
||||||
unsigned int n_lf = 0;
|
unsigned int n_lf = 0;
|
||||||
bool crlf = true;
|
bool crlf = true;
|
||||||
for (size_t i = 0; i < LINE_ENDING_COUNT; i++)
|
for (size_t i = 0; i < LineEndings::COUNT; i++)
|
||||||
{
|
{
|
||||||
lines[i] = std::make_shared<std::list<Span>>();
|
lines[i] = std::make_shared<std::list<Span>>();
|
||||||
}
|
}
|
||||||
@ -30,17 +30,17 @@ void TextLoader::load_buffer(uint8_t * buffer, size_t size)
|
|||||||
{
|
{
|
||||||
if (buffer[i] == '\r')
|
if (buffer[i] == '\r')
|
||||||
{
|
{
|
||||||
lines[LINE_ENDING_CR]->push_back(Span(&buffer[line_start[LINE_ENDING_CR]], i - line_start[LINE_ENDING_CR]));
|
lines[LineEndings::CR]->push_back(Span(&buffer[line_start[LineEndings::CR]], i - line_start[LineEndings::CR]));
|
||||||
n_cr++;
|
n_cr++;
|
||||||
line_start[LINE_ENDING_CR] = i + 1;
|
line_start[LineEndings::CR] = i + 1;
|
||||||
if (crlf)
|
if (crlf)
|
||||||
{
|
{
|
||||||
if ((i < (size - 1)) && (buffer[i + 1] == '\n'))
|
if ((i < (size - 1)) && (buffer[i + 1] == '\n'))
|
||||||
{
|
{
|
||||||
lines[LINE_ENDING_CRLF]->push_back(Span(&buffer[line_start[LINE_ENDING_CRLF]], i - line_start[LINE_ENDING_CRLF]));
|
lines[LineEndings::CRLF]->push_back(Span(&buffer[line_start[LineEndings::CRLF]], i - line_start[LineEndings::CRLF]));
|
||||||
n_lf++;
|
n_lf++;
|
||||||
i++;
|
i++;
|
||||||
line_start[LINE_ENDING_CRLF] = i + 1;
|
line_start[LineEndings::CRLF] = i + 1;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
@ -50,24 +50,24 @@ void TextLoader::load_buffer(uint8_t * buffer, size_t size)
|
|||||||
}
|
}
|
||||||
else if (buffer[i] == '\n')
|
else if (buffer[i] == '\n')
|
||||||
{
|
{
|
||||||
lines[LINE_ENDING_LF]->push_back(Span(&buffer[line_start[LINE_ENDING_LF]], i - line_start[LINE_ENDING_LF]));
|
lines[LineEndings::LF]->push_back(Span(&buffer[line_start[LineEndings::LF]], i - line_start[LineEndings::LF]));
|
||||||
crlf = false;
|
crlf = false;
|
||||||
n_lf++;
|
n_lf++;
|
||||||
line_start[LINE_ENDING_LF] = i + 1;
|
line_start[LineEndings::LF] = i + 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (crlf && (n_lf > 0u))
|
if (crlf && (n_lf > 0u))
|
||||||
{
|
{
|
||||||
m_line_endings = LINE_ENDING_CRLF;
|
m_line_endings = LineEndings::CRLF;
|
||||||
}
|
}
|
||||||
else if ((n_cr > 0u) && (n_lf == 0u))
|
else if ((n_cr > 0u) && (n_lf == 0u))
|
||||||
{
|
{
|
||||||
m_line_endings = LINE_ENDING_CR;
|
m_line_endings = LineEndings::CR;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
m_line_endings = LINE_ENDING_LF;
|
m_line_endings = LineEndings::LF;
|
||||||
}
|
}
|
||||||
|
|
||||||
m_lines = lines[m_line_endings];
|
m_lines = lines[m_line_endings];
|
||||||
|
@ -5,18 +5,11 @@
|
|||||||
#include <list>
|
#include <list>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include "Span.h"
|
#include "Span.h"
|
||||||
|
#include "LineEndings.h"
|
||||||
|
|
||||||
class TextLoader
|
class TextLoader
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
enum
|
|
||||||
{
|
|
||||||
LINE_ENDING_LF,
|
|
||||||
LINE_ENDING_CR,
|
|
||||||
LINE_ENDING_CRLF,
|
|
||||||
LINE_ENDING_COUNT
|
|
||||||
};
|
|
||||||
|
|
||||||
TextLoader();
|
TextLoader();
|
||||||
void load_buffer(uint8_t * buffer, size_t size);
|
void load_buffer(uint8_t * buffer, size_t size);
|
||||||
size_t num_lines()
|
size_t num_lines()
|
||||||
@ -30,13 +23,13 @@ public:
|
|||||||
return m_lines->size();
|
return m_lines->size();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
int get_line_endings() { return m_line_endings; }
|
LineEndings::Type get_line_endings() { return m_line_endings; }
|
||||||
auto begin() { return m_lines->begin(); }
|
auto begin() { return m_lines->begin(); }
|
||||||
auto end() { return m_lines->end(); }
|
auto end() { return m_lines->end(); }
|
||||||
bool get_eol_at_eof() { return m_eol_at_eof; }
|
bool get_eol_at_eof() { return m_eol_at_eof; }
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
int m_line_endings;
|
LineEndings::Type m_line_endings;
|
||||||
bool m_eol_at_eof;
|
bool m_eol_at_eof;
|
||||||
std::shared_ptr<std::list<Span>> m_lines;
|
std::shared_ptr<std::list<Span>> m_lines;
|
||||||
};
|
};
|
||||||
|
@ -31,7 +31,7 @@ TEST(TextLoaderTest, detects_lf_line_endings)
|
|||||||
TextLoader tl;
|
TextLoader tl;
|
||||||
auto file = TestSupport::read_file("test/files/line_endings/lf_format.txt");
|
auto file = TestSupport::read_file("test/files/line_endings/lf_format.txt");
|
||||||
tl.load_buffer(&(*file)[0], file->size());
|
tl.load_buffer(&(*file)[0], file->size());
|
||||||
EXPECT_EQ(TextLoader::LINE_ENDING_LF, tl.get_line_endings());
|
EXPECT_EQ(LineEndings::LF, tl.get_line_endings());
|
||||||
ASSERT_EQ(2u, tl.num_lines());
|
ASSERT_EQ(2u, tl.num_lines());
|
||||||
auto it = tl.begin();
|
auto it = tl.begin();
|
||||||
EXPECT_EQ("Hello.", line_to_string(it));
|
EXPECT_EQ("Hello.", line_to_string(it));
|
||||||
@ -45,7 +45,7 @@ TEST(TextLoaderTest, detects_cr_line_endings)
|
|||||||
TextLoader tl;
|
TextLoader tl;
|
||||||
auto file = TestSupport::read_file("test/files/line_endings/cr_format.txt");
|
auto file = TestSupport::read_file("test/files/line_endings/cr_format.txt");
|
||||||
tl.load_buffer(&(*file)[0], file->size());
|
tl.load_buffer(&(*file)[0], file->size());
|
||||||
EXPECT_EQ(TextLoader::LINE_ENDING_CR, tl.get_line_endings());
|
EXPECT_EQ(LineEndings::CR, tl.get_line_endings());
|
||||||
ASSERT_EQ(2u, tl.num_lines());
|
ASSERT_EQ(2u, tl.num_lines());
|
||||||
auto it = tl.begin();
|
auto it = tl.begin();
|
||||||
EXPECT_EQ("Hello.", line_to_string(it));
|
EXPECT_EQ("Hello.", line_to_string(it));
|
||||||
@ -59,7 +59,7 @@ TEST(TextLoaderTest, detects_crlf_line_endings)
|
|||||||
TextLoader tl;
|
TextLoader tl;
|
||||||
auto file = TestSupport::read_file("test/files/line_endings/crlf_format.txt");
|
auto file = TestSupport::read_file("test/files/line_endings/crlf_format.txt");
|
||||||
tl.load_buffer(&(*file)[0], file->size());
|
tl.load_buffer(&(*file)[0], file->size());
|
||||||
EXPECT_EQ(TextLoader::LINE_ENDING_CRLF, tl.get_line_endings());
|
EXPECT_EQ(LineEndings::CRLF, tl.get_line_endings());
|
||||||
ASSERT_EQ(2u, tl.num_lines());
|
ASSERT_EQ(2u, tl.num_lines());
|
||||||
auto it = tl.begin();
|
auto it = tl.begin();
|
||||||
EXPECT_EQ("Hello.", line_to_string(it));
|
EXPECT_EQ("Hello.", line_to_string(it));
|
||||||
|
Loading…
x
Reference in New Issue
Block a user