======
File path: ./Cargo.toml
[package]
name = "shiva"
version = "0.1.1"
edition = "2021"
authors = ["Evgeny Igumnov <igumnovnsk@gmail.com>"]
repository = "https://github.com/igumnoff/shiva"
documentation = "https://docs.rs/shiva"
license-file = "../LICENSE"
keywords = ["parser"]
categories = ["parser-implementations"]
readme = "../README.md"
description = "Shiva library: Implementation in Rust of a parser and generator for documents of any type"
[package.metadata.docs.rs]
features = ["text", "markdown", "html", "pdf"]
[dependencies]
anyhow = "1.0.75"
bytes = "1.5.0"
thiserror = "1.0.44"
regex = { version = "1.10.3", optional = true }
scraper = { version = "0.19.0", optional = true }
ego-tree = { version = "0.6.2", optional = true }
lopdf = { version = "0.32.0", optional = true }
printpdf = { version = "0.7.0", optional = true }
[dev-dependencies]
env_logger = "0.10.0"
[features]
default = ["all"]
all = ["text", "markdown", "html", "pdf"]
text = []
markdown = ["regex"]
html = ["scraper", "ego-tree"]
pdf = ["lopdf", "printpdf"]
======
File path: ./src/core.rs
use std::fmt::Debug;
use bytes::Bytes;
use thiserror::Error;
use std::any::Any;
use std::collections::HashMap;
pub struct Document {
pub elements: Vec<Box<dyn Element>>,
}
impl Document {
pub fn new(elements: &Vec<Box<dyn Element>>) -> anyhow::Result<Document> {
Ok(Document {
elements: (&elements).to_vec(),
})
}
}
pub trait TransformerTrait {
fn parse(document: &Bytes, images: &HashMap<String, Bytes>) -> anyhow::Result<Document>;
fn generate(document: &Document) -> anyhow::Result<(Bytes, HashMap<String, Bytes>)>;
}
pub enum ParserError {
Common,
}
pub enum GeneratorError {
Common,
}
pub enum CastingError {
Common,
}
pub trait Element: CloneableElement + Debug {
fn as_any(&self) -> &dyn Any;
fn element_type(&self) -> ElementType;
}
pub trait CloneableElement {
fn clone_box(&self) -> Box<dyn Element>;
}
impl<T> CloneableElement for T
where
T: 'static + Element + Clone,
{
fn clone_box(&self) -> Box<dyn Element> {
Box::new(self.clone())
}
}
impl Clone for Box<dyn Element> {
fn clone(&self) -> Box<dyn Element> {
self.clone_box()
}
}
pub enum ElementType {
Text,
Paragraph,
Image,
Hyperlink,
Header,
Table,
TableHeader,
TableRow,
TableCell,
List,
ListItem,
PageBreak,
TableOfContents,
}
pub struct TextElement {
pub text: String,
}
impl TextElement {
pub fn from(element: &Box<dyn Element>) -> anyhow::Result<&TextElement> {
Ok(element.as_any().downcast_ref::<TextElement>().ok_or(CastingError::Common)?)
}
pub fn new(text: &str) -> anyhow::Result<Box<dyn Element>> {
Ok(Box::new(TextElement {
text: text.to_string(),
}))
}
}
impl Element for TextElement {
fn as_any(&self) -> &dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::Text
}
}
pub struct HeaderElement {
pub level: u8,
pub text: String,
pub children: Vec<Box<dyn Element>>,
}
impl HeaderElement {
pub fn from(element: &Box<dyn Element>) -> anyhow::Result<&HeaderElement> {
Ok(element.as_any().downcast_ref::<HeaderElement>().ok_or(CastingError::Common)?)
}
pub fn new(text: &str, level: u8) -> anyhow::Result<Box<dyn Element>> where Self: Sized {
Ok(Box::new(HeaderElement {
level,
text: text.to_string(),
children: vec![],
}))
}
}
impl Element for HeaderElement {
fn as_any(&self) -> &dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::Header
}
}
pub struct ParagraphElement {
pub elements: Vec<Box<dyn Element>>,
}
impl ParagraphElement {
pub fn from(element: &Box<dyn Element>) -> anyhow::Result<&ParagraphElement> {
Ok(element.as_any().downcast_ref::<ParagraphElement>().ok_or(CastingError::Common)?)
}
pub fn new(elements: &Vec<Box<dyn Element>>) -> anyhow::Result<Box<dyn Element>> where Self: Sized {
Ok(Box::new(ParagraphElement {
elements: (&elements).to_vec(),
}))
}
}
impl Element for ParagraphElement {
fn as_any(&self) -> &dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::Paragraph
}
}
pub struct TableElement {
pub rows: Vec<TableRowElement>,
pub headers: Vec<TableHeaderElement>,
}
impl TableElement {
pub fn from(element: &Box<dyn Element>) -> anyhow::Result<&TableElement> {
Ok(element.as_any().downcast_ref::<TableElement>().ok_or(CastingError::Common)?)
}
pub fn new(headers: &Vec<TableHeaderElement>, rows: &Vec<TableRowElement>) -> anyhow::Result<Box<dyn Element>> where Self: Sized {
Ok(Box::new(TableElement {
rows: rows.to_vec(),
headers: headers.to_vec(),
}))
}
}
impl Element for TableElement {
fn as_any(&self) -> &dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::Table
}
}
pub struct TableHeaderElement {
pub element: Box<dyn Element>,
}
impl TableHeaderElement {
pub fn from(element: &Box<dyn Element>) -> anyhow::Result<&TableHeaderElement> {
Ok(element.as_any().downcast_ref::<TableHeaderElement>().ok_or(CastingError::Common)?)
}
pub fn new(element: &Box<dyn Element>) -> anyhow::Result<TableHeaderElement> {
Ok(TableHeaderElement {
element: element.clone(),
})
}
}
impl Element for TableHeaderElement {
fn as_any(&self) -> &dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::TableHeader
}
}
pub struct TableRowElement {
pub cells: Vec<TableCellElement>,
}
impl TableRowElement {
pub fn from(element: &Box<dyn Element>) -> anyhow::Result<&TableRowElement> {
Ok(element.as_any().downcast_ref::<TableRowElement>().ok_or(CastingError::Common)?)
}
pub fn new(cells: &Vec<TableCellElement>) -> anyhow::Result<TableRowElement> {
Ok(TableRowElement {
cells: cells.to_vec(),
})
}
}
impl Element for TableRowElement {
fn as_any(&self) -> &dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::TableRow
}
}
pub struct TableCellElement {
pub element: Box<dyn Element>,
}
impl TableCellElement {
pub fn from(element: &Box<dyn Element>) -> anyhow::Result<&TableCellElement> {
Ok(element.as_any().downcast_ref::<TableCellElement>().ok_or(CastingError::Common)?)
}
pub fn new(element: &Box<dyn Element>) -> anyhow::Result<TableCellElement> {
Ok(TableCellElement {
element: element.clone(),
})
}
}
impl Element for TableCellElement {
fn as_any(&self) -> &dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::TableCell
}
}
pub struct ListElement {
pub elements: Vec<ListItemElement>,
pub numbered: bool,
}
impl ListElement {
pub fn from(element: &Box<dyn Element>) -> anyhow::Result<&ListElement> {
Ok(element.as_any().downcast_ref::<ListElement>().ok_or(CastingError::Common)?)
}
pub fn new(elements: &Vec<ListItemElement>, numbered: bool) -> anyhow::Result<Box<dyn Element>> where Self: Sized {
Ok(Box::new(ListElement {
elements: elements.to_vec(),
numbered,
}))
}
}
impl Element for ListElement {
fn as_any(&self) -> &dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::List
}
}
pub struct ListItemElement {
pub element: Box<dyn Element>,
}
impl ListItemElement {
pub fn from(element: &Box<dyn Element>) -> anyhow::Result<&ListItemElement> {
Ok(element.as_any().downcast_ref::<ListItemElement>().ok_or(CastingError::Common)?)
}
pub fn new(element: &Box<dyn Element>) -> anyhow::Result<ListItemElement> {
Ok(ListItemElement {
element: element.clone(),
})
}
}
impl Element for ListItemElement {
fn as_any(&self) -> &dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::ListItem
}
}
pub enum ImageType {
Png,
Jpeg,
}
pub struct ImageElement {
pub bytes: Bytes,
pub title: String,
pub alt: String,
pub image_type: ImageType,
}
impl ImageElement {
pub fn from(element: &Box<dyn Element>) -> anyhow::Result<&ImageElement> {
Ok(element.as_any().downcast_ref::<ImageElement>().ok_or(CastingError::Common)?)
}
pub fn new(bytes: &Bytes, title: &str, alt: &str, image_type: ImageType) -> anyhow::Result<Box<dyn Element>> where Self: Sized {
Ok(Box::new(ImageElement {
bytes: bytes.clone(),
title: title.to_string(),
alt: alt.to_string(),
image_type,
}))
}
}
impl Element for ImageElement {
fn as_any(&self) -> &dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::Image
}
}
pub struct HyperlinkElement {
pub title: String,
pub url: String,
pub alt: String,
}
impl HyperlinkElement {
pub fn from(element: &Box<dyn Element>) -> anyhow::Result<&HyperlinkElement> {
Ok(element.as_any().downcast_ref::<HyperlinkElement>().ok_or(CastingError::Common)?)
}
pub fn new(title: &str, url: &str, alt: &str) -> anyhow::Result<Box<dyn Element>> where Self: Sized {
Ok(Box::new(HyperlinkElement {
title: title.to_string(),
url: url.to_string(),
alt: alt.to_string(),
}))
}
}
impl Element for HyperlinkElement {
fn as_any(&self) -> &dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::Hyperlink
}
}
======
File path: ./src/lib.rs
pub mod core;
pub mod text;
pub mod markdown;
pub mod html;
pub mod pdf;
======
File path: ./src/markdown.rs
use std::collections::HashMap;
use bytes::Bytes;
use regex::Regex;
use crate::core::*;
pub struct Transformer;
impl TransformerTrait for Transformer {
fn parse(document: &Bytes, images: &HashMap<String, Bytes>) -> anyhow::Result<Document> where Self: Sized {
let document_str = std::str::from_utf8(document)?;
let mut elements: Vec<Box<dyn Element>> = Vec::new();
let lines = document_str.lines();
let mut current_paragraph_elements: Vec<Box<dyn Element>> = Vec::new();
let mut in_table = false;
let mut table_rows: Vec<TableRowElement> = Vec::new();
let mut table_headers: Vec<TableHeaderElement> = Vec::new();
let mut current_list_stack: Vec<Vec<ListItemElement>> = Vec::new();
let mut current_list_type_stack: Vec<bool> = Vec::new();
let img_regex = Regex::new("!\\[.*?\\]\\(.*? \".*?\"\\)").unwrap();
for line in lines {
let indent_level = line.chars().take_while(|c| c.is_whitespace()).count() / 2;
let trimmed_line = line.trim_start();
if trimmed_line.starts_with('-') || trimmed_line.starts_with('+') || trimmed_line.starts_with('*') {
let list_item_text = trimmed_line[1..].trim();
let list_item = ListItemElement::new(&TextElement::new(list_item_text)?).unwrap();
if current_list_stack.len() <= indent_level {
while current_list_stack.len() <= indent_level {
current_list_stack.push(vec![]);
current_list_type_stack.push(false);
}
} else {
while current_list_stack.len() > indent_level + 1 {
let nested_items = current_list_stack.pop().unwrap();
let nested_list = ListElement::new(&nested_items, current_list_type_stack.pop().unwrap())?;
if let Some(parent_list) = current_list_stack.last_mut() {
parent_list.push(ListItemElement::new(&nested_list)?);
}
}
}
current_list_stack[indent_level].push(list_item);
while current_list_stack.len() > indent_level + 1 {
let nested_items = current_list_stack.pop().unwrap();
let nested_list = ListElement::new(&nested_items, current_list_type_stack.pop().unwrap())?;
if let Some(parent_list) = current_list_stack.last_mut() {
parent_list.push(ListItemElement::new(&nested_list)?);
}
}
} else if trimmed_line.chars().next().unwrap_or(' ').is_digit(10) && trimmed_line.chars().nth(1).unwrap_or(' ') == '.' {
let list_item_text = trimmed_line.splitn(2, '.').nth(1).unwrap_or("").trim();
let list_item = ListItemElement::new(&TextElement::new(list_item_text)?).unwrap();
if current_list_stack.len() <= indent_level {
while current_list_stack.len() <= indent_level {
current_list_stack.push(vec![]);
current_list_type_stack.push(trimmed_line.chars().next().unwrap().is_digit(10));
}
} else {
while current_list_stack.len() > indent_level + 1 {
let nested_items = current_list_stack.pop().unwrap();
let nested_list = ListElement::new(&nested_items, current_list_type_stack.pop().unwrap())?;
if let Some(parent_list) = current_list_stack.last_mut() {
parent_list.push(ListItemElement::new(&nested_list)?);
}
}
}
current_list_stack[indent_level].push(list_item);
while current_list_stack.len() > indent_level + 1 {
let nested_items = current_list_stack.pop().unwrap();
let nested_list = ListElement::new(&nested_items, current_list_type_stack.pop().unwrap())?;
if let Some(parent_list) = current_list_stack.last_mut() {
parent_list.push(ListItemElement::new(&nested_list)?);
}
}
} else {
if line.starts_with('|')  && line.ends_with('|') {
if !in_table {
in_table = true;
table_headers = line.split('|')
.filter(|x| !x.trim().is_empty() && !x.contains('-'))
.map(|header| TableHeaderElement::new(&TextElement::new(header.trim()).unwrap()).unwrap())
.collect();
} else if line.contains("---") {
continue;
} else if line.starts_with('|') && in_table {
let cells = line.split('|')
.filter(|x| !x.trim().is_empty())
.map(|cell| TableCellElement::new(&TextElement::new(cell.trim()).unwrap()).unwrap())
.collect();
table_rows.push(TableRowElement::new(&cells).unwrap());
}
}  else {
if in_table {
elements.push(TableElement::new(&table_headers, &table_rows)?);
table_rows.clear();
table_headers.clear();
in_table = false;
}
if line.starts_with('#') {
let level = line.chars().take_while(|&c| c == '#').count() as u8;
let text = line.trim_start_matches('#').trim();
let header = HeaderElement::new(text, level)?;
if !current_paragraph_elements.is_empty() {
elements.push(ParagraphElement::new(&current_paragraph_elements)?);
current_paragraph_elements.clear();
}
elements.push(header);
} else if !line.trim().is_empty() {
fn parser_text(text: &str, current_paragraph_elements: &mut Vec<Box<dyn Element>>) ->  anyhow::Result<()> {
let mut start = 0;
let mut captures = 0;
let hyperlink_regex = Regex::new("(?:\\[([^\\]]+)\\])?(?:\\(|)(http[s]?:\\/\\/[^\\s\\)]+)(?:\\s\"([^\"]+)\")?\\)?").unwrap();
for cap in hyperlink_regex.captures_iter(text) {
captures += 1;
let (hyperlink_start, end) = (cap.get(0).unwrap().start(), cap.get(0).unwrap().end());
let markdown = &text[hyperlink_start..end];
if hyperlink_start > start {
parser_text(&text[start..hyperlink_start], current_paragraph_elements)?;
}
if markdown.starts_with("h") {
current_paragraph_elements.push(HyperlinkElement::new(markdown, markdown, markdown)?);
} else if markdown.starts_with("[") && markdown.ends_with("\")") {
let start_alt_text = markdown.find("[").unwrap() + 1;
let end_alt_text = markdown.find("]").unwrap();
let start_url_path = markdown.find("(").unwrap() + 1;
let end_url_path = markdown.find(" ").unwrap();
let start_title = markdown.find("\"").unwrap() + 1;
let end_title = markdown.rfind("\"").unwrap();
let alt_text = &markdown[start_alt_text..end_alt_text];
let url = &markdown[start_url_path..end_url_path];
let title = &markdown[start_title..end_title];
current_paragraph_elements.push(HyperlinkElement::new(alt_text, url, title)?);
} else {
let start_alt_text = markdown.find("[").unwrap() + 1;
let end_alt_text = markdown.find("]").unwrap();
let start_url_path = markdown.find("(").unwrap() + 1;
let alt_text = &markdown[start_alt_text..end_alt_text];
let url = &markdown[start_url_path..markdown.len()-1];
current_paragraph_elements.push(HyperlinkElement::new(alt_text, url, alt_text)?);
}
start = end;
}
if captures == 0 {
current_paragraph_elements.push(TextElement::new(text)?);
} else {
if start < text.len() {
parser_text(&text[start..], current_paragraph_elements)?;
}
}
Ok(())
}
let  mut captures= 0;
let mut start = 0;
for cap in img_regex.captures_iter(line) {
captures += 1;
let (img_start, img_end) = (cap.get(0).unwrap().start(), cap.get(0).unwrap().end());
let markdown = &line[img_start..img_end];
let start_alt_text = markdown.find("[").unwrap() + 1;
let end_alt_text = markdown.find("]").unwrap();
let start_file_path = markdown.find("(").unwrap() + 1;
let start_title = markdown.find("\"").unwrap() + 1;
let end_title = markdown.rfind("\"").unwrap();
let alt_text = &markdown[start_alt_text..end_alt_text];
let file_path = &markdown[start_file_path..end_title-1];
let title = &markdown[start_title..end_title];
if img_start > start {
parser_text(&line[start..img_start], &mut current_paragraph_elements)?;
}
let image_empty = Bytes::new();
let image_bytes = images.get(file_path).map_or(&image_empty, |x| x);
current_paragraph_elements.push(ImageElement::new(image_bytes, &title, &alt_text, ImageType::Png)?);
start = img_end;
}
if captures ==0 {
parser_text(&line, &mut current_paragraph_elements)?;
} else {
if start < line.len() {
parser_text(&line[start..], &mut current_paragraph_elements)?;
}
}
} else if !current_paragraph_elements.is_empty() {
elements.push(ParagraphElement::new(&current_paragraph_elements)?);
current_paragraph_elements.clear();
}
}
while !current_list_stack.is_empty() {
let items = current_list_stack.pop().unwrap();
let list = ListElement::new(&items, current_list_type_stack.pop().unwrap())?;
if let Some(parent_list) = current_list_stack.last_mut() {
parent_list.push(ListItemElement::new(&list)?);
} else {
elements.push(list);
}
}
}
}
while !current_list_stack.is_empty() {
let items = current_list_stack.pop().unwrap();
let list = ListElement::new(&items, current_list_type_stack.pop().unwrap())?;
if let Some(parent_list) = current_list_stack.last_mut() {
parent_list.push(ListItemElement::new(&list)?);
} else {
elements.push(list);
}
}
if !current_paragraph_elements.is_empty() {
elements.push(ParagraphElement::new(&current_paragraph_elements)?);
}
if in_table {
elements.push(TableElement::new(&table_headers, &table_rows)?);
}
Ok(Document { elements })
}
fn generate(document: &Document) -> anyhow::Result<(Bytes, HashMap<String, Bytes>)> where Self: Sized {
let mut images:HashMap<String, Bytes> = HashMap::new();
let mut image_num:i32 = 0;
let mut markdown = String::new();
fn generate_element(element: &Box<dyn Element>, markdown: &mut String, list_depth: usize, list_counters: &mut Vec<usize>, list_types: &mut Vec<bool>, images: &mut HashMap<String, Bytes>, image_num: &mut i32) -> anyhow::Result<()> {
fn generate_list_item(element:&ListItemElement, markdown: &mut String, list_depth: usize, list_counters: &mut Vec<usize>, list_types: &mut Vec<bool>,  images: &mut HashMap<String, Bytes>, image_num: &mut i32) -> anyhow::Result<()> {
let prefix = if *list_types.last().unwrap() {
let counter = list_counters.last_mut().unwrap();
if &element.element.element_type() == &ElementType::Text {
*counter += 1;
}
format!("{}. ", counter)
} else {
"- ".to_string()
};
markdown.push_str(&"  ".repeat(list_depth-1));
if &element.element.element_type() == &ElementType::Text {
markdown.push_str(&prefix);
}
generate_element(&element.element, markdown, list_depth, list_counters, list_types, images, image_num)?;
if &element.element.element_type() == &ElementType::Text{
markdown.push('\n');
}
Ok(())
}
match element.element_type() {
ElementType::Header => {
let header = HeaderElement::from(element)?;
markdown.push_str(&"#".repeat(header.level as usize));
markdown.push(' ');
markdown.push_str(&header.text);
markdown.push('\n');
markdown.push('\n');
},
ElementType::Paragraph => {
let paragraph = ParagraphElement::from(element)?;
for child in &paragraph.elements {
generate_element(child, markdown, list_depth, list_counters, list_types, images, image_num)?;
}
markdown.push('\n');
markdown.push('\n');
},
ElementType::List => {
let list = ListElement::from(element)?;
list_counters.push(0);
list_types.push(list.numbered);
for item in &list.elements {
generate_list_item(&item, markdown, list_depth + 1, list_counters, list_types, images, image_num)?;
}
list_counters.pop();
list_types.pop();
if list_counters.len() == 0 {
markdown.push('\n');
}
},
ElementType::Text => {
let text = TextElement::from(element)?;
markdown.push_str(&text.text);
if !text.text.ends_with(" ") {
markdown.push_str(" ");
}
},
ElementType::Hyperlink => {
let hyperlink = HyperlinkElement::from(element)?;
if hyperlink.url ==hyperlink.alt && hyperlink.alt == hyperlink.url {
markdown.push_str(&format!("{}", hyperlink.url));
} else {
markdown.push_str(&format!("[{}]({} \"{}\")", hyperlink.title, hyperlink.url, hyperlink.alt));
}
},
ElementType::Image => {
let image = ImageElement::from(element)?;
let image_path = format!("image{}.png", image_num);
markdown.push_str(&format!(
"![{}]({} \"{}\")",
image.alt, image_path, image.title
));
images.insert(image_path.to_string(), image.bytes.clone());
*image_num += 1;
}
ElementType::Table => {
let table = TableElement::from(element)?;
let mut max_lengths: Vec<usize> = Vec::new();
for header in &table.headers {
let header_text = TextElement::from(&header.element)?;
max_lengths.push(header_text.text.len());
}
for row in &table.rows {
for (cell_index, cell) in row.cells.iter().enumerate() {
let cell_text = TextElement::from(&cell.element)?;
if cell_index < max_lengths.len() {
max_lengths[cell_index] = max_lengths[cell_index].max(cell_text.text.len());
}
}
}
for (index, header) in table.headers.iter().enumerate() {
let header_text = TextElement::from(&header.element)?;
let padding = max_lengths[index] - header_text.text.len();
markdown.push_str("| ");
markdown.push_str(&header_text.text);
markdown.push_str(&" ".repeat(padding));
markdown.push(' ');
}
markdown.push_str("|\n");
for max_length in &max_lengths {
markdown.push_str("|");
markdown.push_str(&"-".repeat(*max_length + 2));
}
markdown.push_str("|\n");
for row in &table.rows {
for (cell_index, cell) in row.cells.iter().enumerate() {
let cell_text = TextElement::from(&cell.element)?;
let padding = max_lengths[cell_index] - cell_text.text.len();
markdown.push_str("| ");
markdown.push_str(&cell_text.text);
markdown.push_str(&" ".repeat(padding));
markdown.push(' ');
}
markdown.push_str("|\n");
}
markdown.push('\n');
},
_ =>  {
},
}
Ok(())
}
let mut list_counters: Vec<usize> = Vec::new();
let mut list_types: Vec<bool> = Vec::new();
for element in &document.elements {
generate_element(element, &mut markdown, 0, &mut list_counters,&mut list_types, &mut images, &mut image_num)?;
}
Ok((Bytes::from(markdown), HashMap::new()))
}
}
mod tests {
use crate::core::*;
use crate::markdown::*;
use crate::text;
fn test() -> anyhow::Result<()> {
let document = r#"# First header
Paragraph  bla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla
blabla bla bla blabla bla bla blabla bla bla blabla bla bla bla
1. List item 1
2. List item 2
3. List item 3
1. List item secode level 1
2. List item secode level 2
4. List item 4
1. List item secode level 3
2. List item secode level 4
5. List item 5
1. List item secode level 5
- List item one
- List item two
- List item three
- List item four
- List item five
- List item zzz
- List item six
- List item seven
![Picture alt1](test/data/picture.png "Picture title1")
Bla bla bla ![Picture alt2](test/data/picture.png "Picture title2") bla. http://example.com  [Example](http://example.com) [Example](http://example.com "Example tooltip")
| Syntax      | Description |
| ----------- | ----------- |
| Header      | Title       |
| Paragraph   | Text        |
Paragraph2  bla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla
blabla2 bla bla blabla bla bla blabla bla bla blabla bla bla bla"#;
let mut images = HashMap::new();
let image_bytes = std::fs::read("test/data/picture.png")?;
images.insert("test/data/picture.png".to_string(), image_bytes);
let parsed =  Transformer::parse(&document.as_bytes().into(), &HashMap::new());
let document_string = std::str::from_utf8(document.as_bytes())?;
println!("{}", document_string);
assert!(parsed.is_ok());
let parsed_document = parsed.unwrap();
println!("==========================");
println!("{:?}", parsed_document);
println!("==========================");
let generated_result = Transformer::generate(&parsed_document);
assert!(generated_result.is_ok());
let generated_bytes = generated_result?;
let generated_text = std::str::from_utf8(&generated_bytes.0)?;
println!("{}", generated_text);
println!("==========================");
let generated_result = text::Transformer::generate(&parsed_document);
assert!(generated_result.is_ok());
let generated_bytes = generated_result?;
let generated_text = std::str::from_utf8(&generated_bytes.0)?;
println!("{}", generated_text);
Ok(())
}
}
======
File path: ./src/pdf.rs
use std::collections::{BTreeMap, HashMap};
use bytes::Bytes;
use crate::core::{Document, Element, ElementType, ListElement, ListItemElement, ParagraphElement, ParserError, TextElement, TransformerTrait};
use lopdf::{Document as PdfDocument, Object, ObjectId};
use lopdf::content::Content;
pub struct Transformer;
impl TransformerTrait for Transformer {
fn parse(document: &Bytes, _images: &HashMap<String, Bytes>) -> anyhow::Result<Document> {
let mut elements: Vec<Box<dyn Element>> = Vec::new();
let pdf_document = PdfDocument::load_mem(&document)?;
for (_id, page_id) in pdf_document.get_pages() {
let objects = pdf_document.get_page_contents(page_id);
for object_id in objects {
let object = pdf_document.get_object(object_id)?;
parse_object(page_id, &pdf_document, &object, &mut elements)?;
}
}
Ok(Document { elements })
}
fn generate(_document: &crate::core::Document) -> anyhow::Result<(Bytes, HashMap<String, Bytes>)> {
use printpdf::*;
let (doc, page1, layer1) = PdfDocument::new("PDF_Document_title", Mm(247.0), Mm(210.0), "Layer 1");
let (page2, layer1) = doc.add_page(Mm(10.0), Mm(250.0),"Page 2, Layer 1");
let result = doc.save_to_bytes()?;
let bytes = Bytes::from(result);
Ok((bytes, HashMap::new()))
}
}
fn parse_object(page_id: ObjectId, pdf_document: &PdfDocument, _object: &Object, elements: &mut Vec<Box<dyn Element>>) -> anyhow::Result<()> {
fn collect_text(text: &mut String, encoding: Option<&str>, operands: &[Object], elements: &mut Vec<Box<dyn Element>>) -> anyhow::Result<()>{
for operand in operands.iter() {
match *operand {
Object::String(ref bytes, _) => {
let decoded_text = PdfDocument::decode_text(encoding, bytes);
text.push_str(&decoded_text);
if bytes.len() == 1 && bytes[0] == 1 {
match elements.last() {
None => {
let list_element = ListElement{
elements: vec![],
numbered: false,
};
elements.push(Box::new(list_element));
}
Some(el) => {
if el.element_type() == ElementType::List {
let old_list = elements.pop().unwrap();
let list = ListElement::from(&old_list)?;
let mut list_item_elements = list.elements.clone();
let text_element = TextElement{
text: text.clone(),
};
let new_list_item_element = ListItemElement{
element: Box::new(text_element),
};
list_item_elements.push(new_list_item_element);
let new_list = ListElement{
elements: list_item_elements,
numbered: list.numbered,
};
elements.push(Box::new(new_list));
text.clear();
} else {
if el.element_type() == ElementType::Paragraph {
let old_paragraph = elements.pop().unwrap();
let paragraph = ParagraphElement::from(&old_paragraph)?;
let mut paragraph_elements = paragraph.elements.clone();
let text_element = TextElement{
text: text.clone(),
};
paragraph_elements.push(Box::new(text_element));
let new_paragraph = ParagraphElement{
elements: paragraph_elements,
};
elements.push(Box::new(new_paragraph));
text.clear();
}
let list_element = ListElement{
elements: vec![],
numbered: false,
};
elements.push(Box::new(list_element));
}
}
}
}
}
Object::Array(ref arr) => {
let _ = collect_text(text, encoding, arr, elements);
text.push(' ');
}
Object::Integer(i) => {
if i < -100 {
text.push(' ');
}
}
_ => {}
}
}
Ok(())
}
let mut text = String::new();
let fonts = pdf_document.get_page_fonts(page_id);
let encodings = fonts
.into_iter()
.map(|(name, font)| (name, font.get_font_encoding()))
.collect::<BTreeMap<Vec<u8>, &str>>();
let vec = pdf_document.get_page_content(page_id)?;
let content = Content::decode(&vec)?;
let mut current_encoding = None;
for operation in &content.operations {
match operation.operator.as_ref() {
"Tm" => {
let text_element = TextElement {
text: text.clone(),
};
match elements.last() {
None => {
let paragraph_element = ParagraphElement{
elements: vec![Box::new(text_element)],
};
elements.push(Box::new(paragraph_element));
}
Some(el) => {
if el.element_type() == ElementType::Paragraph {
let old_paragraph = elements.pop().unwrap();
let paragraph = ParagraphElement::from(&old_paragraph)?;
let mut paragraph_elements = paragraph.elements.clone();
paragraph_elements.push(Box::new(text_element));
let new_paragraph = ParagraphElement{
elements: paragraph_elements,
};
elements.push(Box::new(new_paragraph));
} else {
elements.push(Box::new(text_element));
}
}
}
text.clear();
}
"Tf" => {
let current_font = operation
.operands
.first()
.ok_or_else(|| ParserError::Common)?
.as_name()?;
current_encoding = encodings.get(current_font).cloned();
}
"Tj" | "TJ" => {
_ = collect_text(&mut text, current_encoding, &operation.operands, elements);
}
"ET" => {
if !text.ends_with('\n') {
text.push('\n')
}
}
_ => {}
}
}
if text.len() > 0 {
let text_element = TextElement {
text: text.clone(),
};
match elements.last() {
None => {
let paragraph_element = ParagraphElement {
elements: vec![Box::new(text_element)],
};
elements.push(Box::new(paragraph_element));
}
Some(el) => {
if el.element_type() == ElementType::Paragraph {
let old_paragraph = elements.pop().unwrap();
let paragraph = ParagraphElement::from(&old_paragraph)?;
let mut paragraph_elements = paragraph.elements.clone();
paragraph_elements.push(Box::new(text_element));
let new_paragraph = ParagraphElement {
elements: paragraph_elements,
};
elements.push(Box::new(new_paragraph));
} else if el.element_type() == ElementType::List {
let old_list = elements.pop().unwrap();
let list = ListElement::from(&old_list)?;
let mut list_item_elements = list.elements.clone();
let new_list_item_element = ListItemElement{
element: Box::new(text_element),
};
list_item_elements.push(new_list_item_element);
let new_list = ListElement{
elements: list_item_elements,
numbered: list.numbered,
};
elements.push(Box::new(new_list));
} else {
}
}
}
}
println!("{}", text);
Ok(())
}
mod tests {
use std::collections::HashMap;
use bytes::Bytes;
use crate::core::*;
use crate::pdf::Transformer;
fn test() -> anyhow::Result<()> {
let pdf = std::fs::read("test/data/document.pdf")?;
let pdf_bytes = Bytes::from(pdf);
let parsed =  Transformer::parse(&pdf_bytes, &HashMap::new());
assert!(parsed.is_ok());
let parsed_document = parsed.unwrap();
println!("==========================");
println!("{:?}", parsed_document);
println!("==========================");
Ok(())
}
}
