mirror of https://github.com/realaravinth/gitpad
feat: read_file processes directories
SUMMARY This program supports directories in gists. This patch modifies Data::read_file to support directory reads. Additionally, CreateGistRequest is modified to accept files in subdirectories. DESCRIPTION Data::read_file When repository contains subdirectories, it will recursively read all files and return their contents. crate::data::api::v1::gists::GitFileMode Set of known(to me) file modes that Git uses. GitFileMode::Unsupported is used to take advantage of num_enum::FromPrimitive. The alternative would have been num_enum::TryFromPrimitive, which returns errors on unsupported values. I felt the former was cleaner.master
parent
311964f32f
commit
632fe272fa
|
@ -1054,6 +1054,7 @@ dependencies = [
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
"log",
|
"log",
|
||||||
"num_cpus",
|
"num_cpus",
|
||||||
|
"num_enum",
|
||||||
"pretty_env_logger",
|
"pretty_env_logger",
|
||||||
"rand 0.8.4",
|
"rand 0.8.4",
|
||||||
"serde 1.0.136",
|
"serde 1.0.136",
|
||||||
|
@ -1597,6 +1598,27 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num_enum"
|
||||||
|
version = "0.5.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "720d3ea1055e4e4574c0c0b0f8c3fd4f24c4cdaf465948206dea090b57b526ad"
|
||||||
|
dependencies = [
|
||||||
|
"num_enum_derive",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num_enum_derive"
|
||||||
|
version = "0.5.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "0d992b768490d7fe0d8586d9b5745f6c49f557da6d81dc982b1d167ad4edbb21"
|
||||||
|
dependencies = [
|
||||||
|
"proc-macro-crate",
|
||||||
|
"proc-macro2",
|
||||||
|
"quote",
|
||||||
|
"syn",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num_threads"
|
name = "num_threads"
|
||||||
version = "0.1.3"
|
version = "0.1.3"
|
||||||
|
@ -1793,6 +1815,16 @@ dependencies = [
|
||||||
"log",
|
"log",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "proc-macro-crate"
|
||||||
|
version = "1.1.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "9dada8c9981fcf32929c3c0f0cd796a9284aca335565227ed88c83babb1d43dc"
|
||||||
|
dependencies = [
|
||||||
|
"thiserror",
|
||||||
|
"toml",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "proc-macro-error"
|
name = "proc-macro-error"
|
||||||
version = "1.0.4"
|
version = "1.0.4"
|
||||||
|
|
|
@ -47,6 +47,7 @@ tokio = { version = "1.16.1", features = ["fs"] }
|
||||||
url = "2.2"
|
url = "2.2"
|
||||||
urlencoding = "2.1.0"
|
urlencoding = "2.1.0"
|
||||||
validator = { version = "0.14.0", features = ["derive"] }
|
validator = { version = "0.14.0", features = ["derive"] }
|
||||||
|
num_enum = "0.5.6"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
actix-rt = "2"
|
actix-rt = "2"
|
||||||
|
|
|
@ -20,16 +20,22 @@ use actix_web::*;
|
||||||
use db_core::prelude::*;
|
use db_core::prelude::*;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use crate::data::api::v1::gists::{CreateGist, File, GistID};
|
use crate::data::api::v1::gists::{CreateGist, FileInfo, GistID};
|
||||||
use crate::errors::*;
|
use crate::errors::*;
|
||||||
use crate::*;
|
use crate::*;
|
||||||
|
|
||||||
|
//#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||||
|
//pub struct File {
|
||||||
|
// pub filename: String,
|
||||||
|
// pub content: ContentType,
|
||||||
|
//}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct CreateGistRequest {
|
pub struct CreateGistRequest {
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub description: Option<String>,
|
pub description: Option<String>,
|
||||||
pub visibility: GistVisibility,
|
pub visibility: GistVisibility,
|
||||||
pub files: Vec<File>,
|
pub files: Vec<FileInfo>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CreateGistRequest {
|
impl CreateGistRequest {
|
||||||
|
@ -74,7 +80,7 @@ async fn new(
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::data::api::v1::gists::ContentType;
|
use crate::data::api::v1::gists::{ContentType, FileType};
|
||||||
use crate::tests::*;
|
use crate::tests::*;
|
||||||
|
|
||||||
#[actix_rt::test]
|
#[actix_rt::test]
|
||||||
|
@ -96,17 +102,17 @@ mod tests {
|
||||||
let app = get_app!(data, db).await;
|
let app = get_app!(data, db).await;
|
||||||
|
|
||||||
let files = [
|
let files = [
|
||||||
File {
|
FileInfo {
|
||||||
filename: "foo".into(),
|
filename: "foo".into(),
|
||||||
content: ContentType::Text("foobar".into()),
|
content: FileType::File(ContentType::Text("foobar".into())),
|
||||||
},
|
},
|
||||||
File {
|
FileInfo {
|
||||||
filename: "bar".into(),
|
filename: "bar".into(),
|
||||||
content: ContentType::Text("foobar".into()),
|
content: FileType::File(ContentType::Text("foobar".into())),
|
||||||
},
|
},
|
||||||
File {
|
FileInfo {
|
||||||
filename: "foo bar".into(),
|
filename: "foo bar".into(),
|
||||||
content: ContentType::Text("foobar".into()),
|
content: FileType::File(ContentType::Text("foobar".into())),
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
|
|
@ -18,6 +18,7 @@ use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
use db_core::prelude::*;
|
use db_core::prelude::*;
|
||||||
use git2::*;
|
use git2::*;
|
||||||
|
use num_enum::FromPrimitive;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
|
|
||||||
|
@ -25,6 +26,53 @@ use crate::errors::*;
|
||||||
use crate::utils::*;
|
use crate::utils::*;
|
||||||
use crate::*;
|
use crate::*;
|
||||||
|
|
||||||
|
/// A FileMode represents the kind of tree entries used by git. It
|
||||||
|
/// resembles regular file systems modes, although FileModes are
|
||||||
|
/// considerably simpler (there are not so many), and there are some,
|
||||||
|
/// like Submodule that has no file system equivalent.
|
||||||
|
// Adapted from https://github.com/go-git/go-git/blob/master/plumbing/filemode/filemode.go(Apache-2.0 License)
|
||||||
|
#[derive(Debug, PartialEq, Clone, FromPrimitive)]
|
||||||
|
#[repr(isize)]
|
||||||
|
pub enum GitFileMode {
|
||||||
|
/// Empty is used as the GitFileMode of tree elements when comparing
|
||||||
|
/// trees in the following situations:
|
||||||
|
///
|
||||||
|
/// - the mode of tree elements before their creation.
|
||||||
|
/// - the mode of tree elements after their deletion.
|
||||||
|
/// - the mode of unmerged elements when checking the index.
|
||||||
|
///
|
||||||
|
/// Empty has no file system equivalent. As Empty is the zero value
|
||||||
|
/// of [GitFileMode]
|
||||||
|
Empty = 0,
|
||||||
|
/// Regular represent non-executable files.
|
||||||
|
Regular = 0o100644,
|
||||||
|
/// Dir represent a Directory.
|
||||||
|
Dir = 0o40000,
|
||||||
|
/// Deprecated represent non-executable files with the group writable bit set. This mode was
|
||||||
|
/// supported by the first versions of git, but it has been deprecated nowadays. This
|
||||||
|
/// library(github.com/go-git/go-git uses it, not realaravinth/gists at the moment) uses them
|
||||||
|
/// internally, so you can read old packfiles, but will treat them as Regulars when interfacing
|
||||||
|
/// with the outside world. This is the standard git behaviour.
|
||||||
|
Deprecated = 0o100664,
|
||||||
|
/// Executable represents executable files.
|
||||||
|
Executable = 0o100755,
|
||||||
|
/// Symlink represents symbolic links to files.
|
||||||
|
Symlink = 0o120000,
|
||||||
|
/// Submodule represents git submodules. This mode has no file system
|
||||||
|
/// equivalent.
|
||||||
|
Submodule = 0o160000,
|
||||||
|
|
||||||
|
/// Unsupported file mode
|
||||||
|
#[num_enum(default)]
|
||||||
|
Unsupported = -1,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl From<&'_ TreeEntry<'_>> for GitFileMode {
|
||||||
|
fn from(t: &TreeEntry) -> Self {
|
||||||
|
GitFileMode::from(t.filemode() as isize)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub struct Gist {
|
pub struct Gist {
|
||||||
pub id: String,
|
pub id: String,
|
||||||
pub repository: git2::Repository,
|
pub repository: git2::Repository,
|
||||||
|
@ -36,17 +84,26 @@ pub struct CreateGist<'a> {
|
||||||
pub visibility: &'a GistVisibility,
|
pub visibility: &'a GistVisibility,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
|
||||||
pub struct File {
|
|
||||||
pub filename: String,
|
|
||||||
pub content: ContentType,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub enum GistID<'a> {
|
pub enum GistID<'a> {
|
||||||
Repository(&'a mut git2::Repository),
|
Repository(&'a mut git2::Repository),
|
||||||
ID(&'a str),
|
ID(&'a str),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
|
||||||
|
pub struct FileInfo {
|
||||||
|
pub filename: String,
|
||||||
|
pub content: FileType,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct GistInfo {
|
||||||
|
pub files: Vec<FileInfo>,
|
||||||
|
pub description: String,
|
||||||
|
pub owner: String,
|
||||||
|
pub created: i64,
|
||||||
|
pub updated: i64,
|
||||||
|
pub visibility: GistVisibility,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Serialize, PartialEq, Clone, Debug, Deserialize)]
|
#[derive(Serialize, PartialEq, Clone, Debug, Deserialize)]
|
||||||
#[serde(rename_all = "lowercase")]
|
#[serde(rename_all = "lowercase")]
|
||||||
pub enum ContentType {
|
pub enum ContentType {
|
||||||
|
@ -71,12 +128,12 @@ impl ContentType {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
|
||||||
#[serde(rename_all = "lowercase")]
|
#[serde(rename_all = "lowercase")]
|
||||||
pub enum FileType {
|
pub enum FileType {
|
||||||
/// Contains file content
|
/// Contains file content
|
||||||
File(ContentType),
|
File(ContentType),
|
||||||
Dir,
|
Dir(Vec<FileInfo>),
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Data {
|
impl Data {
|
||||||
|
@ -126,7 +183,7 @@ impl Data {
|
||||||
&self,
|
&self,
|
||||||
_db: &T,
|
_db: &T,
|
||||||
gist_id: GistID<'_>,
|
gist_id: GistID<'_>,
|
||||||
files: &[File],
|
files: &[FileInfo],
|
||||||
) -> ServiceResult<()> {
|
) -> ServiceResult<()> {
|
||||||
// TODO change updated in DB
|
// TODO change updated in DB
|
||||||
let inner = |repo: &mut Repository| -> ServiceResult<()> {
|
let inner = |repo: &mut Repository| -> ServiceResult<()> {
|
||||||
|
@ -136,13 +193,16 @@ impl Data {
|
||||||
for file in files.iter() {
|
for file in files.iter() {
|
||||||
let escaped_filename = escape_spaces(&file.filename);
|
let escaped_filename = escape_spaces(&file.filename);
|
||||||
|
|
||||||
let obj = odb
|
match &file.content {
|
||||||
.write(ObjectType::Blob, file.content.as_bytes())
|
FileType::Dir(dir_contents) => unimplemented!(),
|
||||||
.unwrap();
|
FileType::File(f) => {
|
||||||
|
let obj = odb.write(ObjectType::Blob, f.as_bytes()).unwrap();
|
||||||
tree_builder
|
tree_builder
|
||||||
.insert(&escaped_filename, obj, 0o100644)
|
.insert(&escaped_filename, obj, 0o100644)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let tree_hash = tree_builder.write().unwrap();
|
let tree_hash = tree_builder.write().unwrap();
|
||||||
let author = Signature::now("gists", "admin@gists.batsense.net").unwrap();
|
let author = Signature::now("gists", "admin@gists.batsense.net").unwrap();
|
||||||
|
@ -178,14 +238,10 @@ impl Data {
|
||||||
match gist_id {
|
match gist_id {
|
||||||
GistID::ID(path) => {
|
GistID::ID(path) => {
|
||||||
let mut repo = git2::Repository::open(self.get_repository_path(path)).unwrap();
|
let mut repo = git2::Repository::open(self.get_repository_path(path)).unwrap();
|
||||||
inner(&mut repo)?;
|
inner(&mut repo)
|
||||||
}
|
}
|
||||||
GistID::Repository(repository) => {
|
GistID::Repository(repository) => inner(repository),
|
||||||
inner(repository)?;
|
|
||||||
}
|
}
|
||||||
};
|
|
||||||
|
|
||||||
Ok(())
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Please note that this method expects path to not contain any spaces
|
/// Please note that this method expects path to not contain any spaces
|
||||||
|
@ -197,15 +253,73 @@ impl Data {
|
||||||
pub async fn read_file<T: GistDatabase>(
|
pub async fn read_file<T: GistDatabase>(
|
||||||
&self,
|
&self,
|
||||||
_db: &T,
|
_db: &T,
|
||||||
gist_id: &str,
|
gist_id: GistID<'_>,
|
||||||
path: &str,
|
path: &str,
|
||||||
) -> ServiceResult<ContentType> {
|
) -> ServiceResult<FileInfo> {
|
||||||
let repo = git2::Repository::open(self.get_repository_path(gist_id)).unwrap();
|
let inner = |repo: &git2::Repository| -> ServiceResult<FileInfo> {
|
||||||
let head = repo.head().unwrap();
|
let head = repo.head().unwrap();
|
||||||
let tree = head.peel_to_tree().unwrap();
|
let tree = head.peel_to_tree().unwrap();
|
||||||
let entry = tree.get_path(Path::new(path)).unwrap();
|
let entry = tree.get_path(Path::new(path)).unwrap();
|
||||||
let blob = repo.find_blob(entry.id()).unwrap();
|
GitFileMode::Regular as i32;
|
||||||
Ok(ContentType::from_blob(&blob))
|
|
||||||
|
fn read_file(id: Oid, repo: &git2::Repository) -> FileType {
|
||||||
|
let blob = repo.find_blob(id).unwrap();
|
||||||
|
FileType::File(ContentType::from_blob(&blob))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_dir(id: Oid, repo: &Repository) -> FileType {
|
||||||
|
let tree = repo.find_tree(id).unwrap();
|
||||||
|
let mut items = Vec::with_capacity(tree.len());
|
||||||
|
for item in tree.iter() {
|
||||||
|
println!("{:?}", &item.name());
|
||||||
|
if let Some(name) = item.name() {
|
||||||
|
let mode: GitFileMode = (&item).into();
|
||||||
|
let file = match mode {
|
||||||
|
GitFileMode::Dir => read_dir(item.id(), repo),
|
||||||
|
GitFileMode::Submodule => unimplemented!(),
|
||||||
|
GitFileMode::Empty => unimplemented!(),
|
||||||
|
GitFileMode::Deprecated => unimplemented!(),
|
||||||
|
GitFileMode::Unsupported => unimplemented!(),
|
||||||
|
GitFileMode::Symlink => unimplemented!(),
|
||||||
|
GitFileMode::Executable => read_file(item.id(), repo),
|
||||||
|
GitFileMode::Regular => read_file(item.id(), repo),
|
||||||
|
};
|
||||||
|
items.push(FileInfo {
|
||||||
|
filename: name.to_owned(),
|
||||||
|
content: file,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
FileType::Dir(items)
|
||||||
|
}
|
||||||
|
let mode: GitFileMode = (&entry).into();
|
||||||
|
if let Some(name) = entry.name() {
|
||||||
|
let file = match mode {
|
||||||
|
GitFileMode::Dir => read_dir(entry.id(), repo),
|
||||||
|
GitFileMode::Submodule => unimplemented!(),
|
||||||
|
GitFileMode::Empty => unimplemented!(),
|
||||||
|
GitFileMode::Deprecated => unimplemented!(),
|
||||||
|
GitFileMode::Unsupported => unimplemented!(),
|
||||||
|
GitFileMode::Symlink => unimplemented!(),
|
||||||
|
GitFileMode::Executable => read_file(entry.id(), repo),
|
||||||
|
GitFileMode::Regular => read_file(entry.id(), repo),
|
||||||
|
};
|
||||||
|
Ok(FileInfo {
|
||||||
|
filename: name.to_string(),
|
||||||
|
content: file,
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
unimplemented!();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
match gist_id {
|
||||||
|
GistID::ID(path) => {
|
||||||
|
let repo = git2::Repository::open(self.get_repository_path(path)).unwrap();
|
||||||
|
inner(&repo)
|
||||||
|
}
|
||||||
|
GistID::Repository(repository) => inner(repository),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -233,14 +347,18 @@ pub mod tests {
|
||||||
&self,
|
&self,
|
||||||
db: &T,
|
db: &T,
|
||||||
gist_id: &str,
|
gist_id: &str,
|
||||||
files: &[File],
|
files: &[FileInfo],
|
||||||
) {
|
) {
|
||||||
for file in files.iter() {
|
for file in files.iter() {
|
||||||
let content = self
|
let content = self
|
||||||
.read_file(db, &gist_id, &escape_spaces(&file.filename))
|
.read_file(db, GistID::ID(&gist_id), &escape_spaces(&file.filename))
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(content, file.content);
|
let req_escaped_file = FileInfo {
|
||||||
|
filename: escape_spaces(&file.filename),
|
||||||
|
content: file.content.clone(),
|
||||||
|
};
|
||||||
|
assert_eq!(&content, &req_escaped_file);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -272,17 +390,17 @@ pub mod tests {
|
||||||
|
|
||||||
// save files
|
// save files
|
||||||
let files = [
|
let files = [
|
||||||
File {
|
FileInfo {
|
||||||
filename: "foo".into(),
|
filename: "foo".into(),
|
||||||
content: ContentType::Text("foobar".into()),
|
content: FileType::File(ContentType::Text("foobar".into())),
|
||||||
},
|
},
|
||||||
File {
|
FileInfo {
|
||||||
filename: "bar".into(),
|
filename: "bar".into(),
|
||||||
content: ContentType::Text("foobar".into()),
|
content: FileType::File(ContentType::Text("foobar".into())),
|
||||||
},
|
},
|
||||||
File {
|
FileInfo {
|
||||||
filename: "foo bar".into(),
|
filename: "foo bar".into(),
|
||||||
content: ContentType::Text("foobar".into()),
|
content: FileType::File(ContentType::Text("foobar".into())),
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
@ -290,9 +408,9 @@ pub mod tests {
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
data.gist_files_written_helper(db, &gist.id, &files).await;
|
data.gist_files_written_helper(db, &gist.id, &files).await;
|
||||||
let files2 = [File {
|
let files2 = [FileInfo {
|
||||||
filename: "notfirstcommit".into(),
|
filename: "notfirstcommit".into(),
|
||||||
content: ContentType::Text("foobar".into()),
|
content: FileType::File(ContentType::Text("foobar".into())),
|
||||||
}];
|
}];
|
||||||
|
|
||||||
data.write_file(db, GistID::ID(&gist.id), &files2)
|
data.write_file(db, GistID::ID(&gist.id), &files2)
|
||||||
|
|
Loading…
Reference in New Issue