======
File path: ./Cargo.toml
[package]
name = "shiva"
version = "0.1.9"
edition = "2021"
authors = ["Evgeny Igumnov <igumnovnsk@gmail.com>"]
repository = "https://github.com/igumnoff/shiva"
documentation = "https://docs.rs/shiva"
license-file = "../LICENSE"
keywords = ["parser", "conversion", "transformer", "text-processing", "data-conversion"]
categories = [
"parser-implementations",
"text-processing",
"command-line-utilities",
"data-structures",
"data-structures"]
readme = "../README.md"
description = "Shiva library: Implementation in Rust of a parser and generator for documents of any type"
[package.metadata.docs.rs]
features = ["text", "markdown", "html", "pdf"]
[dependencies]
anyhow = "1.0.75"
bytes = "1.5.0"
thiserror = "1.0.44"
regex = { version = "1.10.3", optional = true }
scraper = { version = "0.19.0", optional = true }
ego-tree = { version = "0.6.2", optional = true }
lopdf = { version = "0.32.0", optional = true }
printpdf = { version = "0.7.0", optional = true }
[dev-dependencies]
env_logger = "0.10.0"
[features]
default = ["all"]
all = ["text", "markdown", "html", "pdf"]
text = []
markdown = ["regex"]
html = ["scraper", "ego-tree"]
pdf = ["lopdf", "printpdf"]
======
File path: ./src/core.rs
use bytes::Bytes;
use std::any::Any;
use std::collections::HashMap;
use std::fmt::Debug;
use thiserror::Error;
pub struct Document {
pub elements: Vec<Box<dyn Element>>,
pub page_width: f32,
pub page_height: f32,
pub left_page_indent: f32,
pub right_page_indent: f32,
pub top_page_indent: f32,
pub bottom_page_indent: f32,
}
impl Document {
pub fn new(elements: &Vec<Box<dyn Element>>) -> anyhow::Result<Document> {
Ok(Document {
elements: (&elements).to_vec(),
page_width: 210.0,
page_height: 297.0,
left_page_indent: 10.0,
right_page_indent: 10.0,
top_page_indent: 10.0,
bottom_page_indent: 10.0,
})
}
}
pub trait TransformerTrait {
fn parse(document: &Bytes, images: &HashMap<String, Bytes>) -> anyhow::Result<Document>;
fn generate(document: &Document) -> anyhow::Result<(Bytes, HashMap<String, Bytes>)>;
}
pub enum ParserError {
Common,
}
pub enum GeneratorError {
Common,
}
pub enum CastingError {
Common,
}
pub trait Element: CloneableElement + Debug {
fn as_any(&self) -> &dyn Any;
fn as_any_mut(&mut self) -> &mut dyn Any;
fn element_type(&self) -> ElementType;
fn paragraph_as_ref(&self) -> anyhow::Result<&ParagraphElement> {
Ok(self
.as_any()
.downcast_ref::<ParagraphElement>()
.ok_or(CastingError::Common)?)
}
fn paragraph_as_mut(&mut self) -> anyhow::Result<&mut ParagraphElement> {
Ok(self
.as_any_mut()
.downcast_mut::<ParagraphElement>()
.ok_or(CastingError::Common)?)
}
fn header_as_ref(&self) -> anyhow::Result<&HeaderElement> {
Ok(self
.as_any()
.downcast_ref::<HeaderElement>()
.ok_or(CastingError::Common)?)
}
fn header_as_mut(&mut self) -> anyhow::Result<&mut HeaderElement> {
Ok(self
.as_any_mut()
.downcast_mut::<HeaderElement>()
.ok_or(CastingError::Common)?)
}
fn text_as_ref(&self) -> anyhow::Result<&TextElement> {
Ok(self
.as_any()
.downcast_ref::<TextElement>()
.ok_or(CastingError::Common)?)
}
fn text_as_mut(&mut self) -> anyhow::Result<&mut TextElement> {
Ok(self
.as_any_mut()
.downcast_mut::<TextElement>()
.ok_or(CastingError::Common)?)
}
fn table_as_ref(&self) -> anyhow::Result<&TableElement> {
Ok(self
.as_any()
.downcast_ref::<TableElement>()
.ok_or(CastingError::Common)?)
}
fn table_as_mut(&mut self) -> anyhow::Result<&mut TableElement> {
Ok(self
.as_any_mut()
.downcast_mut::<TableElement>()
.ok_or(CastingError::Common)?)
}
fn table_header_as_ref(&self) -> anyhow::Result<&TableHeaderElement> {
Ok(self
.as_any()
.downcast_ref::<TableHeaderElement>()
.ok_or(CastingError::Common)?)
}
fn table_header_as_mut(&mut self) -> anyhow::Result<&mut TableHeaderElement> {
Ok(self
.as_any_mut()
.downcast_mut::<TableHeaderElement>()
.ok_or(CastingError::Common)?)
}
fn table_row_as_ref(&self) -> anyhow::Result<&TableRowElement> {
Ok(self
.as_any()
.downcast_ref::<TableRowElement>()
.ok_or(CastingError::Common)?)
}
fn table_row_as_mut(&mut self) -> anyhow::Result<&mut TableRowElement> {
Ok(self
.as_any_mut()
.downcast_mut::<TableRowElement>()
.ok_or(CastingError::Common)?)
}
fn table_cell_as_ref(&self) -> anyhow::Result<&TableCellElement> {
Ok(self
.as_any()
.downcast_ref::<TableCellElement>()
.ok_or(CastingError::Common)?)
}
fn table_cell_as_mut(&mut self) -> anyhow::Result<&mut TableCellElement> {
Ok(self
.as_any_mut()
.downcast_mut::<TableCellElement>()
.ok_or(CastingError::Common)?)
}
fn list_as_ref(&self) -> anyhow::Result<&ListElement> {
Ok(self
.as_any()
.downcast_ref::<ListElement>()
.ok_or(CastingError::Common)?)
}
fn list_as_mut(&mut self) -> anyhow::Result<&mut ListElement> {
Ok(self
.as_any_mut()
.downcast_mut::<ListElement>()
.ok_or(CastingError::Common)?)
}
fn list_item_as_ref(&self) -> anyhow::Result<&ListItemElement> {
Ok(self
.as_any()
.downcast_ref::<ListItemElement>()
.ok_or(CastingError::Common)?)
}
fn list_item_as_mut(&mut self) -> anyhow::Result<&mut ListItemElement> {
Ok(self
.as_any_mut()
.downcast_mut::<ListItemElement>()
.ok_or(CastingError::Common)?)
}
fn image_as_ref(&self) -> anyhow::Result<&ImageElement> {
Ok(self
.as_any()
.downcast_ref::<ImageElement>()
.ok_or(CastingError::Common)?)
}
fn image_as_mut(&mut self) -> anyhow::Result<&mut ImageElement> {
Ok(self
.as_any_mut()
.downcast_mut::<ImageElement>()
.ok_or(CastingError::Common)?)
}
fn hyperlink_as_ref(&self) -> anyhow::Result<&HyperlinkElement> {
Ok(self
.as_any()
.downcast_ref::<HyperlinkElement>()
.ok_or(CastingError::Common)?)
}
fn hyperlink_as_mut(&mut self) -> anyhow::Result<&mut HyperlinkElement> {
Ok(self
.as_any_mut()
.downcast_mut::<HyperlinkElement>()
.ok_or(CastingError::Common)?)
}
}
pub trait CloneableElement {
fn clone_box(&self) -> Box<dyn Element>;
}
impl<T> CloneableElement for T
where
T: 'static + Element + Clone,
{
fn clone_box(&self) -> Box<dyn Element> {
Box::new(self.clone())
}
}
impl Clone for Box<dyn Element> {
fn clone(&self) -> Box<dyn Element> {
self.clone_box()
}
}
pub enum ElementType {
Text,
Paragraph,
Image,
Hyperlink,
Header,
Table,
TableHeader,
TableRow,
TableCell,
List,
ListItem,
PageBreak,
TableOfContents,
}
pub struct TextElement {
pub text: String,
pub size: u8,
}
impl TextElement {
pub fn new(text: &str, size: u8) -> anyhow::Result<Box<dyn Element>> {
Ok(Box::new(TextElement {
text: text.to_string(),
size,
}))
}
}
impl Element for TextElement {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::Text
}
}
pub struct HeaderElement {
pub level: u8,
pub text: String,
pub children: Vec<Box<dyn Element>>,
}
impl HeaderElement {
pub fn new(text: &str, level: u8) -> anyhow::Result<Box<dyn Element>> {
Ok(Box::new(HeaderElement {
level,
text: text.to_string(),
children: vec![],
}))
}
}
impl Element for HeaderElement {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::Header
}
}
pub struct ParagraphElement {
pub elements: Vec<Box<dyn Element>>,
}
impl ParagraphElement {
pub fn new(elements: &Vec<Box<dyn Element>>) -> anyhow::Result<Box<dyn Element>> {
Ok(Box::new(ParagraphElement {
elements: (&elements).to_vec(),
}))
}
}
impl Element for ParagraphElement {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::Paragraph
}
}
pub struct TableElement {
pub rows: Vec<TableRowElement>,
pub headers: Vec<TableHeaderElement>,
}
impl TableElement {
pub fn new(
headers: &Vec<TableHeaderElement>,
rows: &Vec<TableRowElement>,
) -> anyhow::Result<Box<dyn Element>> {
Ok(Box::new(TableElement {
rows: rows.to_vec(),
headers: headers.to_vec(),
}))
}
}
impl Element for TableElement {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::Table
}
}
pub struct TableHeaderElement {
pub element: Box<dyn Element>,
}
impl TableHeaderElement {
pub fn new(element: &Box<dyn Element>) -> anyhow::Result<TableHeaderElement> {
Ok(TableHeaderElement {
element: element.clone(),
})
}
}
impl Element for TableHeaderElement {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::TableHeader
}
}
pub struct TableRowElement {
pub cells: Vec<TableCellElement>,
}
impl TableRowElement {
pub fn new(cells: &Vec<TableCellElement>) -> anyhow::Result<TableRowElement> {
Ok(TableRowElement {
cells: cells.to_vec(),
})
}
}
impl Element for TableRowElement {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::TableRow
}
}
pub struct TableCellElement {
pub element: Box<dyn Element>,
}
impl TableCellElement {
pub fn new(element: &Box<dyn Element>) -> anyhow::Result<TableCellElement> {
Ok(TableCellElement {
element: element.clone(),
})
}
}
impl Element for TableCellElement {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::TableCell
}
}
pub struct ListElement {
pub elements: Vec<ListItemElement>,
pub numbered: bool,
}
impl ListElement {
pub fn new(
elements: &Vec<ListItemElement>,
numbered: bool,
) -> anyhow::Result<Box<dyn Element>> {
Ok(Box::new(ListElement {
elements: elements.to_vec(),
numbered,
}))
}
}
impl Element for ListElement {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::List
}
}
pub struct ListItemElement {
pub element: Box<dyn Element>,
}
impl ListItemElement {
pub fn new(element: &Box<dyn Element>) -> anyhow::Result<ListItemElement> {
Ok(ListItemElement {
element: element.clone(),
})
}
}
impl Element for ListItemElement {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::ListItem
}
}
pub enum ImageType {
Png,
Jpeg,
}
pub struct ImageElement {
pub bytes: Bytes,
pub title: String,
pub alt: String,
pub image_type: ImageType,
}
impl ImageElement {
pub fn new(
bytes: &Bytes,
title: &str,
alt: &str,
image_type: ImageType,
) -> anyhow::Result<Box<dyn Element>> {
Ok(Box::new(ImageElement {
bytes: bytes.clone(),
title: title.to_string(),
alt: alt.to_string(),
image_type,
}))
}
}
impl Element for ImageElement {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::Image
}
}
pub struct HyperlinkElement {
pub title: String,
pub url: String,
pub alt: String,
}
impl HyperlinkElement {
pub fn new(title: &str, url: &str, alt: &str) -> anyhow::Result<Box<dyn Element>> {
Ok(Box::new(HyperlinkElement {
title: title.to_string(),
url: url.to_string(),
alt: alt.to_string(),
}))
}
}
impl Element for HyperlinkElement {
fn as_any(&self) -> &dyn Any {
self
}
fn as_any_mut(&mut self) -> &mut dyn Any {
self
}
fn element_type(&self) -> ElementType {
ElementType::Hyperlink
}
}
======
File path: ./src/lib.rs
pub mod core;
pub mod text;
pub mod markdown;
pub mod html;
pub mod pdf;
======
File path: ./src/markdown.rs
use crate::core::*;
use bytes::Bytes;
use regex::Regex;
use std::collections::HashMap;
pub struct Transformer;
impl TransformerTrait for Transformer {
fn parse(document: &Bytes, images: &HashMap<String, Bytes>) -> anyhow::Result<Document>
where
Self: Sized,
{
let document_str = std::str::from_utf8(document)?;
let mut elements: Vec<Box<dyn Element>> = Vec::new();
let lines = document_str.lines();
let mut current_paragraph_elements: Vec<Box<dyn Element>> = Vec::new();
let mut in_table = false;
let mut table_rows: Vec<TableRowElement> = Vec::new();
let mut table_headers: Vec<TableHeaderElement> = Vec::new();
let mut current_list_stack: Vec<Vec<ListItemElement>> = Vec::new();
let mut current_list_type_stack: Vec<bool> = Vec::new();
let img_regex = Regex::new("!\\[.*?\\]\\(.*? \".*?\"\\)").unwrap();
for line in lines {
let indent_level = line.chars().take_while(|c| c.is_whitespace()).count() / 2;
let trimmed_line = line.trim_start();
if trimmed_line.starts_with('-')
|| trimmed_line.starts_with('+')
|| trimmed_line.starts_with('*')
{
let list_item_text = trimmed_line[1..].trim();
let list_item =
ListItemElement::new(&TextElement::new(list_item_text, 8)?).unwrap();
if current_list_stack.len() <= indent_level {
while current_list_stack.len() <= indent_level {
current_list_stack.push(vec![]);
current_list_type_stack.push(false);
}
} else {
while current_list_stack.len() > indent_level + 1 {
let nested_items = current_list_stack.pop().unwrap();
let nested_list = ListElement::new(
&nested_items,
current_list_type_stack.pop().unwrap(),
)?;
if let Some(parent_list) = current_list_stack.last_mut() {
parent_list.push(ListItemElement::new(&nested_list)?);
}
}
}
current_list_stack[indent_level].push(list_item);
while current_list_stack.len() > indent_level + 1 {
let nested_items = current_list_stack.pop().unwrap();
let nested_list =
ListElement::new(&nested_items, current_list_type_stack.pop().unwrap())?;
if let Some(parent_list) = current_list_stack.last_mut() {
parent_list.push(ListItemElement::new(&nested_list)?);
}
}
} else if trimmed_line.chars().next().unwrap_or(' ').is_digit(10)
&& trimmed_line.chars().nth(1).unwrap_or(' ') == '.'
{
let list_item_text = trimmed_line.splitn(2, '.').nth(1).unwrap_or("").trim();
let list_item =
ListItemElement::new(&TextElement::new(list_item_text, 8)?).unwrap();
if current_list_stack.len() <= indent_level {
while current_list_stack.len() <= indent_level {
current_list_stack.push(vec![]);
current_list_type_stack
.push(trimmed_line.chars().next().unwrap().is_digit(10));
}
} else {
while current_list_stack.len() > indent_level + 1 {
let nested_items = current_list_stack.pop().unwrap();
let nested_list = ListElement::new(
&nested_items,
current_list_type_stack.pop().unwrap(),
)?;
if let Some(parent_list) = current_list_stack.last_mut() {
parent_list.push(ListItemElement::new(&nested_list)?);
}
}
}
current_list_stack[indent_level].push(list_item);
while current_list_stack.len() > indent_level + 1 {
let nested_items = current_list_stack.pop().unwrap();
let nested_list =
ListElement::new(&nested_items, current_list_type_stack.pop().unwrap())?;
if let Some(parent_list) = current_list_stack.last_mut() {
parent_list.push(ListItemElement::new(&nested_list)?);
}
}
} else {
if line.starts_with('|') && line.ends_with('|') {
if !in_table {
in_table = true;
table_headers = line
.split('|')
.filter(|x| !x.trim().is_empty() && !x.contains('-'))
.map(|header| {
TableHeaderElement::new(
&TextElement::new(header.trim(), 8).unwrap(),
)
.unwrap()
})
.collect();
} else if line.contains("---") {
continue;
} else if line.starts_with('|') && in_table {
let cells = line
.split('|')
.filter(|x| !x.trim().is_empty())
.map(|cell| {
TableCellElement::new(&TextElement::new(cell.trim(), 8).unwrap())
.unwrap()
})
.collect();
table_rows.push(TableRowElement::new(&cells).unwrap());
}
} else {
if in_table {
elements.push(TableElement::new(&table_headers, &table_rows)?);
table_rows.clear();
table_headers.clear();
in_table = false;
}
if line.starts_with('#') {
let level = line.chars().take_while(|&c| c == '#').count() as u8;
let text = line.trim_start_matches('#').trim();
let header = HeaderElement::new(text, level)?;
if !current_paragraph_elements.is_empty() {
elements.push(ParagraphElement::new(&current_paragraph_elements)?);
current_paragraph_elements.clear();
}
elements.push(header);
} else if !line.trim().is_empty() {
fn parser_text(
text: &str,
current_paragraph_elements: &mut Vec<Box<dyn Element>>,
) -> anyhow::Result<()> {
let mut start = 0;
let mut captures = 0;
let hyperlink_regex = Regex::new("(?:\\[([^\\]]+)\\])?(?:\\(|)(http[s]?:\\/\\/[^\\s\\)]+)(?:\\s\"([^\"]+)\")?\\)?").unwrap();
for cap in hyperlink_regex.captures_iter(text) {
captures += 1;
let (hyperlink_start, end) =
(cap.get(0).unwrap().start(), cap.get(0).unwrap().end());
let markdown = &text[hyperlink_start..end];
if hyperlink_start > start {
parser_text(
&text[start..hyperlink_start],
current_paragraph_elements,
)?;
}
if markdown.starts_with("h") {
current_paragraph_elements
.push(HyperlinkElement::new(markdown, markdown, markdown)?);
} else if markdown.starts_with("[") && markdown.ends_with("\")") {
let start_alt_text = markdown.find("[").unwrap() + 1;
let end_alt_text = markdown.find("]").unwrap();
let start_url_path = markdown.find("(").unwrap() + 1;
let end_url_path = markdown.find(" ").unwrap();
let start_title = markdown.find("\"").unwrap() + 1;
let end_title = markdown.rfind("\"").unwrap();
let alt_text = &markdown[start_alt_text..end_alt_text];
let url = &markdown[start_url_path..end_url_path];
let title = &markdown[start_title..end_title];
current_paragraph_elements
.push(HyperlinkElement::new(alt_text, url, title)?);
} else {
let start_alt_text = markdown.find("[").unwrap() + 1;
let end_alt_text = markdown.find("]").unwrap();
let start_url_path = markdown.find("(").unwrap() + 1;
let alt_text = &markdown[start_alt_text..end_alt_text];
let url = &markdown[start_url_path..markdown.len() - 1];
current_paragraph_elements
.push(HyperlinkElement::new(alt_text, url, alt_text)?);
}
start = end;
}
if captures == 0 {
current_paragraph_elements.push(TextElement::new(text, 8)?);
} else {
if start < text.len() {
parser_text(&text[start..], current_paragraph_elements)?;
}
}
Ok(())
}
let mut captures = 0;
let mut start = 0;
for cap in img_regex.captures_iter(line) {
captures += 1;
let (img_start, img_end) =
(cap.get(0).unwrap().start(), cap.get(0).unwrap().end());
let markdown = &line[img_start..img_end];
let start_alt_text = markdown.find("[").unwrap() + 1;
let end_alt_text = markdown.find("]").unwrap();
let start_file_path = markdown.find("(").unwrap() + 1;
let start_title = markdown.find("\"").unwrap() + 1;
let end_title = markdown.rfind("\"").unwrap();
let alt_text = &markdown[start_alt_text..end_alt_text];
let file_path = &markdown[start_file_path..end_title - 1];
let title = &markdown[start_title..end_title];
if img_start > start {
parser_text(
&line[start..img_start],
&mut current_paragraph_elements,
)?;
}
let image_empty = Bytes::new();
let image_bytes = images.get(file_path).map_or(&image_empty, |x| x);
current_paragraph_elements.push(ImageElement::new(
image_bytes,
&title,
&alt_text,
ImageType::Png,
)?);
start = img_end;
}
if captures == 0 {
parser_text(&line, &mut current_paragraph_elements)?;
} else {
if start < line.len() {
parser_text(&line[start..], &mut current_paragraph_elements)?;
}
}
} else if !current_paragraph_elements.is_empty() {
elements.push(ParagraphElement::new(&current_paragraph_elements)?);
current_paragraph_elements.clear();
}
}
while !current_list_stack.is_empty() {
let items = current_list_stack.pop().unwrap();
let list = ListElement::new(&items, current_list_type_stack.pop().unwrap())?;
if let Some(parent_list) = current_list_stack.last_mut() {
parent_list.push(ListItemElement::new(&list)?);
} else {
elements.push(list);
}
}
}
}
while !current_list_stack.is_empty() {
let items = current_list_stack.pop().unwrap();
let list = ListElement::new(&items, current_list_type_stack.pop().unwrap())?;
if let Some(parent_list) = current_list_stack.last_mut() {
parent_list.push(ListItemElement::new(&list)?);
} else {
elements.push(list);
}
}
if !current_paragraph_elements.is_empty() {
elements.push(ParagraphElement::new(&current_paragraph_elements)?);
}
if in_table {
elements.push(TableElement::new(&table_headers, &table_rows)?);
}
Ok(Document::new(&elements)?)
}
fn generate(document: &Document) -> anyhow::Result<(Bytes, HashMap<String, Bytes>)>
where
Self: Sized,
{
let mut images: HashMap<String, Bytes> = HashMap::new();
let mut image_num: i32 = 0;
let mut markdown = String::new();
fn generate_element(
element: &Box<dyn Element>,
markdown: &mut String,
list_depth: usize,
list_counters: &mut Vec<usize>,
list_types: &mut Vec<bool>,
images: &mut HashMap<String, Bytes>,
image_num: &mut i32,
) -> anyhow::Result<()> {
fn generate_list_item(
element: &ListItemElement,
markdown: &mut String,
list_depth: usize,
list_counters: &mut Vec<usize>,
list_types: &mut Vec<bool>,
images: &mut HashMap<String, Bytes>,
image_num: &mut i32,
) -> anyhow::Result<()> {
let prefix = if *list_types.last().unwrap() {
let counter = list_counters.last_mut().unwrap();
if &element.element.element_type() == &ElementType::Text {
*counter += 1;
}
format!("{}. ", counter)
} else {
"- ".to_string()
};
markdown.push_str(&"  ".repeat(list_depth - 1));
if &element.element.element_type() == &ElementType::Text {
markdown.push_str(&prefix);
}
generate_element(
&element.element,
markdown,
list_depth,
list_counters,
list_types,
images,
image_num,
)?;
if &element.element.element_type() == &ElementType::Text {
markdown.push('\n');
}
Ok(())
}
match element.element_type() {
ElementType::Header => {
let header = element.header_as_ref()?;
markdown.push_str(&"#".repeat(header.level as usize));
markdown.push(' ');
markdown.push_str(&header.text);
markdown.push('\n');
markdown.push('\n');
}
ElementType::Paragraph => {
let paragraph = element.paragraph_as_ref()?;
for child in &paragraph.elements {
generate_element(
child,
markdown,
list_depth,
list_counters,
list_types,
images,
image_num,
)?;
}
markdown.push('\n');
markdown.push('\n');
}
ElementType::List => {
let list = element.list_as_ref()?;
list_counters.push(0);
list_types.push(list.numbered);
for item in &list.elements {
generate_list_item(
&item,
markdown,
list_depth + 1,
list_counters,
list_types,
images,
image_num,
)?;
}
list_counters.pop();
list_types.pop();
if list_counters.len() == 0 {
markdown.push('\n');
}
}
ElementType::Text => {
let text = element.text_as_ref()?;
markdown.push_str(&text.text);
if !text.text.ends_with(" ") {
markdown.push_str(" ");
}
}
ElementType::Hyperlink => {
let hyperlink = element.hyperlink_as_ref()?;
if hyperlink.url == hyperlink.alt && hyperlink.alt == hyperlink.url {
markdown.push_str(&format!("{}", hyperlink.url));
} else {
markdown.push_str(&format!(
"[{}]({} \"{}\")",
hyperlink.title, hyperlink.url, hyperlink.alt
));
}
}
ElementType::Image => {
let image = element.image_as_ref()?;
let image_path = format!("image{}.png", image_num);
markdown.push_str(&format!(
"![{}]({} \"{}\")",
image.alt, image_path, image.title
));
images.insert(image_path.to_string(), image.bytes.clone());
*image_num += 1;
}
ElementType::Table => {
let table = element.table_as_ref()?;
let mut max_lengths: Vec<usize> = Vec::new();
for header in &table.headers {
let header_text = header.element.text_as_ref()?;
max_lengths.push(header_text.text.len());
}
for row in &table.rows {
for (cell_index, cell) in row.cells.iter().enumerate() {
let cell_text = cell.element.text_as_ref()?;
if cell_index < max_lengths.len() {
max_lengths[cell_index] =
max_lengths[cell_index].max(cell_text.text.len());
}
}
}
for (index, header) in table.headers.iter().enumerate() {
let header_text = header.element.text_as_ref()?;
let padding = max_lengths[index] - header_text.text.len();
markdown.push_str("| ");
markdown.push_str(&header_text.text);
markdown.push_str(&" ".repeat(padding));
markdown.push(' ');
}
markdown.push_str("|\n");
for max_length in &max_lengths {
markdown.push_str("|");
markdown.push_str(&"-".repeat(*max_length + 2));
}
markdown.push_str("|\n");
for row in &table.rows {
for (cell_index, cell) in row.cells.iter().enumerate() {
let cell_text = cell.element.text_as_ref()?;
let padding = max_lengths[cell_index] - cell_text.text.len();
markdown.push_str("| ");
markdown.push_str(&cell_text.text);
markdown.push_str(&" ".repeat(padding));
markdown.push(' ');
}
markdown.push_str("|\n");
}
markdown.push('\n');
}
_ => {}
}
Ok(())
}
let mut list_counters: Vec<usize> = Vec::new();
let mut list_types: Vec<bool> = Vec::new();
for element in &document.elements {
generate_element(
element,
&mut markdown,
0,
&mut list_counters,
&mut list_types,
&mut images,
&mut image_num,
)?;
}
Ok((Bytes::from(markdown), HashMap::new()))
}
}
mod tests {
use crate::core::*;
use crate::markdown::*;
use crate::text;
fn test() -> anyhow::Result<()> {
let document = r#"# First header
Paragraph  bla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla
blabla bla bla blabla bla bla blabla bla bla blabla bla bla bla
1. List item 1
2. List item 2
3. List item 3
1. List item secode level 1
2. List item secode level 2
4. List item 4
1. List item secode level 3
2. List item secode level 4
5. List item 5
1. List item secode level 5
- List item one
- List item two
- List item three
- List item four
- List item five
- List item zzz
- List item six
- List item seven
![Picture alt1](test/data/picture.png "Picture title1")
Bla bla bla ![Picture alt2](test/data/picture.png "Picture title2") bla. http://example.com  [Example](http://example.com) [Example](http://example.com "Example tooltip")
| Syntax      | Description |
| ----------- | ----------- |
| Header      | Title       |
| Paragraph   | Text        |
Paragraph2  bla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla blabla bla bla
blabla2 bla bla blabla bla bla blabla bla bla blabla bla bla bla"#;
let mut images = HashMap::new();
let image_bytes = std::fs::read("test/data/picture.png")?;
images.insert("test/data/picture.png".to_string(), image_bytes);
let parsed = Transformer::parse(&document.as_bytes().into(), &HashMap::new());
let document_string = std::str::from_utf8(document.as_bytes())?;
println!("{}", document_string);
assert!(parsed.is_ok());
let parsed_document = parsed.unwrap();
println!("==========================");
println!("{:?}", parsed_document);
println!("==========================");
let generated_result = Transformer::generate(&parsed_document);
assert!(generated_result.is_ok());
let generated_bytes = generated_result?;
let generated_text = std::str::from_utf8(&generated_bytes.0)?;
println!("{}", generated_text);
println!("==========================");
let generated_result = text::Transformer::generate(&parsed_document);
assert!(generated_result.is_ok());
let generated_bytes = generated_result?;
let generated_text = std::str::from_utf8(&generated_bytes.0)?;
println!("{}", generated_text);
Ok(())
}
}
======
File path: ./src/pdf.rs
use crate::core::{
Document, Element, ElementType, ListElement, ListItemElement, ParagraphElement, ParserError,
TextElement, TransformerTrait,
};
use bytes::Bytes;
use lopdf::content::Content;
use lopdf::{Document as PdfDocument, Object, ObjectId};
use std::collections::{BTreeMap, HashMap};
pub struct Transformer;
impl TransformerTrait for Transformer {
fn parse(document: &Bytes, _images: &HashMap<String, Bytes>) -> anyhow::Result<Document> {
let mut elements: Vec<Box<dyn Element>> = Vec::new();
let pdf_document = PdfDocument::load_mem(&document)?;
for (_id, page_id) in pdf_document.get_pages() {
let objects = pdf_document.get_page_contents(page_id);
for object_id in objects {
let object = pdf_document.get_object(object_id)?;
parse_object(page_id, &pdf_document, &object, &mut elements)?;
}
}
Ok(Document::new(&elements)?)
}
fn generate(document: &Document) -> anyhow::Result<(Bytes, HashMap<String, Bytes>)> {
use printpdf::*;
const PAGE_WIDTH: f32 = 210.0;
const PAGE_HEIGHT: f32 = 297.0;
let (mut pdf, page1, layer1) =
PdfDocument::new("PDF Document", Mm(PAGE_WIDTH), Mm(PAGE_HEIGHT), "Layer 1");
fn generate_pdf(document: &Document,
pdf: &mut PdfDocumentReference,
mut page: PdfPageIndex,
mut layer: PdfLayerIndex,
mut vertical_position: f32,
) -> anyhow::Result<()> {
for element in &document.elements {
match element.as_ref() {
e if e.element_type() == ElementType::Header => {
let header = element.header_as_ref()?;
let font_size = match header.level {
1 => 18.0, // Example font size for level 1 header
2 => 16.0, // Example font size for level 2 header
3 => 14.0, // Example font size for level 3 header
_ => 12.0, // Default font size for other header levels
};
let font_width = (0.3528 * (font_size as f32) * 0.87) as f32;
let max_text_width = document.page_height - document.left_page_indent - document.right_page_indent;
let max_chars = (max_text_width  / font_width) as usize ;
let text_elements = split_string(&header.text, max_chars);
for text in text_elements {
let step: f32 = 0.3528 * font_size as f32;
if (vertical_position + step) > (document.page_height - document.bottom_page_indent) {
let (new_page, new_layer) = pdf.add_page(
Mm(document.page_width),
Mm(document.page_height),
"Layer 1",
);
vertical_position = 0.0 + document.top_page_indent;
layer = new_layer;
page = new_page;
}
vertical_position = vertical_position + step;
let font = pdf.add_builtin_font(BuiltinFont::Courier)?;
let current_layer = pdf.get_page(page).get_layer(layer);
current_layer.use_text(
text,
font_size as f32,
Mm(document.left_page_indent + 0.0),
Mm(document.page_height - vertical_position),
&font,
);
vertical_position = vertical_position + 2.5;
}
}
e if e.element_type() == ElementType::Paragraph => {
let paragraph = element.paragraph_as_ref()?;
for paragraph_element in &paragraph.elements {
match paragraph_element.as_ref() {
e if e.element_type() == ElementType::Text => {
let text_element = paragraph_element.text_as_ref()?;
let font_width = (0.3528 * (text_element.size as f32) * 0.87) as f32;
let max_text_width = document.page_height - document.left_page_indent - document.right_page_indent;
let max_chars = (max_text_width  / font_width) as usize ;
let text_elements = split_string(&text_element.text, max_chars);
for text in text_elements {
let step: f32 = 0.3528 * text_element.size as f32;
if (vertical_position + step) > (document.page_height - document.bottom_page_indent) {
let (new_page, new_layer) = pdf.add_page(
Mm(document.page_width),
Mm(document.page_height),
"Layer 1",
);
vertical_position = 0.0 + document.top_page_indent;
layer = new_layer;
page = new_page;
}
vertical_position = vertical_position + step;
let font = pdf.add_builtin_font(BuiltinFont::Courier)?;
let current_layer = pdf.get_page(page).get_layer(layer);
current_layer.use_text(
text,
text_element.size as f32,
Mm(document.left_page_indent + 0.0),
Mm(document.page_height - vertical_position),
&font,
);
}
}
_ => {}
}
}
}
_ => {}
}
}
Ok(())
}
_ = generate_pdf(document, &mut pdf, page1, layer1, 0.0  + document.top_page_indent)?;
let result = pdf.save_to_bytes()?;
let bytes = Bytes::from(result);
Ok((bytes, HashMap::new()))
}
}
fn parse_object(
page_id: ObjectId,
pdf_document: &PdfDocument,
_object: &Object,
elements: &mut Vec<Box<dyn Element>>,
) -> anyhow::Result<()> {
fn collect_text(
text: &mut String,
encoding: Option<&str>,
operands: &[Object],
elements: &mut Vec<Box<dyn Element>>,
) -> anyhow::Result<()> {
for operand in operands.iter() {
match *operand {
Object::String(ref bytes, _) => {
let decoded_text = PdfDocument::decode_text(encoding, bytes);
text.push_str(&decoded_text);
if bytes.len() == 1 && bytes[0] == 1 {
match elements.last() {
None => {
let list_element = ListElement {
elements: vec![],
numbered: false,
};
elements.push(Box::new(list_element));
}
Some(el) => {
if el.element_type() == ElementType::List {
let old_list = elements.pop().unwrap();
let list = old_list.list_as_ref()?;
let mut list_item_elements = list.elements.clone();
let text_element = TextElement {
text: text.clone(),
size: 8,
};
let new_list_item_element = ListItemElement {
element: Box::new(text_element),
};
list_item_elements.push(new_list_item_element);
let new_list = ListElement {
elements: list_item_elements,
numbered: list.numbered,
};
elements.push(Box::new(new_list));
text.clear();
} else {
if el.element_type() == ElementType::Paragraph {
let old_paragraph = elements.pop().unwrap();
let paragraph = old_paragraph.paragraph_as_ref()?;
let mut paragraph_elements = paragraph.elements.clone();
let text_element = TextElement {
text: text.clone(),
size: 8,
};
paragraph_elements.push(Box::new(text_element));
let new_paragraph = ParagraphElement {
elements: paragraph_elements,
};
elements.push(Box::new(new_paragraph));
text.clear();
}
let list_element = ListElement {
elements: vec![],
numbered: false,
};
elements.push(Box::new(list_element));
}
}
}
}
}
Object::Array(ref arr) => {
let _ = collect_text(text, encoding, arr, elements);
text.push(' ');
}
Object::Integer(i) => {
if i < -100 {
text.push(' ');
}
}
_ => {}
}
}
Ok(())
}
let mut text = String::new();
let fonts = pdf_document.get_page_fonts(page_id);
let encodings = fonts
.into_iter()
.map(|(name, font)| (name, font.get_font_encoding()))
.collect::<BTreeMap<Vec<u8>, &str>>();
let vec = pdf_document.get_page_content(page_id)?;
let content = Content::decode(&vec)?;
let mut current_encoding = None;
for operation in &content.operations {
match operation.operator.as_ref() {
"Tm" => {
let text_element = TextElement {
text: text.clone(),
size: 8,
};
match elements.last() {
None => {
let paragraph_element = ParagraphElement {
elements: vec![Box::new(text_element)],
};
elements.push(Box::new(paragraph_element));
}
Some(el) => {
if el.element_type() == ElementType::Paragraph {
let old_paragraph = elements.pop().unwrap();
let paragraph = old_paragraph.paragraph_as_ref()?;
let mut paragraph_elements = paragraph.elements.clone();
paragraph_elements.push(Box::new(text_element));
let new_paragraph = ParagraphElement {
elements: paragraph_elements,
};
elements.push(Box::new(new_paragraph));
} else {
elements.push(Box::new(text_element));
}
}
}
text.clear();
}
"Tf" => {
let current_font = operation
.operands
.first()
.ok_or_else(|| ParserError::Common)?
.as_name()?;
current_encoding = encodings.get(current_font).cloned();
}
"Tj" | "TJ" => {
_ = collect_text(&mut text, current_encoding, &operation.operands, elements);
}
"ET" => {
if !text.ends_with('\n') {
text.push('\n')
}
}
_ => {}
}
}
if text.len() > 0 {
let text_element = TextElement {
text: text.clone(),
size: 8,
};
match elements.last() {
None => {
let paragraph_element = ParagraphElement {
elements: vec![Box::new(text_element)],
};
elements.push(Box::new(paragraph_element));
}
Some(el) => {
if el.element_type() == ElementType::Paragraph {
let old_paragraph = elements.pop().unwrap();
let paragraph = old_paragraph.paragraph_as_ref()?;
let mut paragraph_elements = paragraph.elements.clone();
paragraph_elements.push(Box::new(text_element));
let new_paragraph = ParagraphElement {
elements: paragraph_elements,
};
elements.push(Box::new(new_paragraph));
} else if el.element_type() == ElementType::List {
let old_list = elements.pop().unwrap();
let list = old_list.list_as_ref()?;
let mut list_item_elements = list.elements.clone();
let new_list_item_element = ListItemElement {
element: Box::new(text_element),
};
list_item_elements.push(new_list_item_element);
let new_list = ListElement {
elements: list_item_elements,
numbered: list.numbered,
};
elements.push(Box::new(new_list));
} else {
}
}
}
}
println!("{}", text);
Ok(())
}
fn split_string(input: &str, max_length: usize) -> Vec<String> {
let mut result = Vec::new();
let mut current_string = String::new();
for char in input.chars() {
if current_string.chars().count() < max_length {
current_string.push(char);
} else {
result.push(current_string);
current_string = char.to_string();
}
}
if !current_string.is_empty() {
result.push(current_string);
}
result
}
mod tests {
use crate::core::*;
use crate::pdf::Transformer;
use bytes::Bytes;
use std::collections::HashMap;
fn test() -> anyhow::Result<()> {
let pdf = std::fs::read("test/data/document.pdf")?;
let pdf_bytes = Bytes::from(pdf);
let parsed = Transformer::parse(&pdf_bytes, &HashMap::new());
assert!(parsed.is_ok());
let parsed_document = parsed.unwrap();
println!("==========================");
println!("{:?}", parsed_document);
println!("==========================");
Ok(())
}
}
