Skip to content

Commit b7ff0df

Browse files
mkleenmkleen
committed
Fix linter
1 parent 96464a3 commit b7ff0df

7 files changed

Lines changed: 72 additions & 36 deletions

File tree

‎datafusion-cli/src/main.rs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -443,10 +443,7 @@ mod tests {
443443
use super::*;
444444
use datafusion::{
445445
common::test_util::batches_to_string,
446-
execution::cache::{
447-
DefaultListFilesCache, cache_manager::CacheManagerConfig,
448-
cache_unit::DefaultFileStatisticsCache,
449-
},
446+
execution::cache::{DefaultListFilesCache, cache_manager::CacheManagerConfig},
450447
prelude::{ParquetReadOptions, col, lit, split_part},
451448
};
452449
use insta::assert_snapshot;

‎datafusion/catalog-listing/src/helpers.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -359,7 +359,7 @@ fn try_into_partitioned_file(
359359

360360
let mut pf: PartitionedFile = object_meta.into();
361361
pf.partition_values = partition_values;
362-
pf.table_reference = table_path.get_table_ref().clone();
362+
pf.table_reference.clone_from(table_path.get_table_ref());
363363
Ok(pf)
364364
}
365365

‎datafusion/catalog-listing/src/table.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -804,7 +804,10 @@ impl ListingTable {
804804
) -> datafusion_common::Result<(Arc<Statistics>, Option<LexOrdering>)> {
805805
use datafusion_execution::cache::cache_manager::CachedFileMetadata;
806806

807-
let path = TableScopedPath { table: part_file.table_reference.clone(), path : part_file.object_meta.location.clone()};
807+
let path = TableScopedPath {
808+
table: part_file.table_reference.clone(),
809+
path: part_file.object_meta.location.clone(),
810+
};
808811
let meta = &part_file.object_meta;
809812

810813
// Check cache first - if we have valid cached statistics and ordering

‎datafusion/core/src/execution/context/mod.rs‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1184,7 +1184,7 @@ impl SessionContext {
11841184
builder.with_object_list_cache_ttl(Some(duration))
11851185
}
11861186
"file_statistics_cache_limit" => {
1187-
let limit = Self::parse_memory_limit(value)?;
1187+
let limit = Self::parse_capacity_limit(variable, value)?;
11881188
builder.with_file_statistics_cache_limit(limit)
11891189
}
11901190
_ => return plan_err!("Unknown runtime configuration: {variable}"),
@@ -1427,7 +1427,8 @@ impl SessionContext {
14271427
schema.deregister_table(&table)?;
14281428
if table_type == TableType::Base
14291429
&& let Some(lfc) = self.runtime_env().cache_manager.get_list_files_cache()
1430-
&& let Some(fsc) = self.runtime_env().cache_manager.get_file_statistic_cache()
1430+
&& let Some(fsc) =
1431+
self.runtime_env().cache_manager.get_file_statistic_cache()
14311432
{
14321433
lfc.drop_table_entries(&Some(table_ref.clone()))?;
14331434
fsc.drop_table_entries(&Some(table_ref.clone()))?;
@@ -1763,9 +1764,9 @@ impl SessionContext {
17631764
let config = ListingTableConfig::new(table_path)
17641765
.with_listing_options(options)
17651766
.with_schema(resolved_schema);
1766-
let table = ListingTable::try_new(config)?.with_definition(sql_definition).with_cache(
1767-
self.runtime_env().cache_manager.get_file_statistic_cache()
1768-
);
1767+
let table = ListingTable::try_new(config)?
1768+
.with_definition(sql_definition)
1769+
.with_cache(self.runtime_env().cache_manager.get_file_statistic_cache());
17691770
self.register_table(table_ref, Arc::new(table))?;
17701771
Ok(())
17711772
}

‎datafusion/datasource/src/mod.rs‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ pub use self::url::ListingTableUrl;
5555
use crate::file_groups::FileGroup;
5656
use chrono::TimeZone;
5757
use datafusion_common::stats::Precision;
58-
use datafusion_common::{ColumnStatistics, Result, exec_datafusion_err, TableReference};
58+
use datafusion_common::{ColumnStatistics, Result, TableReference, exec_datafusion_err};
5959
use datafusion_common::{ScalarValue, Statistics};
6060
use datafusion_physical_expr::LexOrdering;
6161
use futures::{Stream, StreamExt};
@@ -171,7 +171,7 @@ impl PartitionedFile {
171171
ordering: None,
172172
extensions: None,
173173
metadata_size_hint: None,
174-
table_reference: None
174+
table_reference: None,
175175
}
176176
}
177177

@@ -185,7 +185,7 @@ impl PartitionedFile {
185185
ordering: None,
186186
extensions: None,
187187
metadata_size_hint: None,
188-
table_reference: None
188+
table_reference: None,
189189
}
190190
}
191191

@@ -205,7 +205,7 @@ impl PartitionedFile {
205205
ordering: None,
206206
extensions: None,
207207
metadata_size_hint: None,
208-
table_reference: None
208+
table_reference: None,
209209
}
210210
.with_range(start, end)
211211
}
@@ -217,12 +217,14 @@ impl PartitionedFile {
217217
self
218218
}
219219

220-
pub fn with_table_reference(mut self, table_reference: Option<TableReference>) -> Self {
220+
pub fn with_table_reference(
221+
mut self,
222+
table_reference: Option<TableReference>,
223+
) -> Self {
221224
self.table_reference = table_reference;
222225
self
223226
}
224227

225-
226228
/// Size of the file to be scanned (taking into account the range, if present).
227229
pub fn effective_size(&self) -> u64 {
228230
if let Some(range) = &self.range {

‎datafusion/execution/src/cache/cache_manager.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,9 @@ impl CachedFileMetadata {
9595
/// 3. If invalid or missing, compute new value and call `put(path, new_value)`
9696
///
9797
/// See [`crate::runtime_env::RuntimeEnv`] for more details
98-
pub trait FileStatisticsCache: CacheAccessor<TableScopedPath, CachedFileMetadata> {
98+
pub trait FileStatisticsCache:
99+
CacheAccessor<TableScopedPath, CachedFileMetadata>
100+
{
99101
/// Cache memory limit in bytes.
100102
fn cache_limit(&self) -> usize;
101103

@@ -106,7 +108,6 @@ pub trait FileStatisticsCache: CacheAccessor<TableScopedPath, CachedFileMetadata
106108
fn list_entries(&self) -> HashMap<Path, FileStatisticsCacheEntry>;
107109

108110
fn drop_table_entries(&self, table_ref: &Option<TableReference>) -> Result<()>;
109-
110111
}
111112

112113
impl DFHeapSize for CachedFileMetadata {

‎datafusion/execution/src/cache/cache_unit.rs‎

Lines changed: 49 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -15,18 +15,18 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use crate::cache::{CacheAccessor, TableScopedPath};
1918
use crate::cache::cache_manager::{
2019
CachedFileMetadata, FileStatisticsCache, FileStatisticsCacheEntry,
2120
};
21+
use crate::cache::{CacheAccessor, TableScopedPath};
2222
use object_store::path::Path;
2323
use std::collections::HashMap;
2424
use std::sync::Mutex;
2525

2626
pub use crate::cache::DefaultFilesMetadataCache;
2727
use crate::cache::lru_queue::LruQueue;
28-
use datafusion_common::heap_size::DFHeapSize;
2928
use datafusion_common::TableReference;
29+
use datafusion_common::heap_size::DFHeapSize;
3030

3131
/// Default implementation of [`FileStatisticsCache`]
3232
///
@@ -175,7 +175,11 @@ impl CacheAccessor<TableScopedPath, CachedFileMetadata> for DefaultFileStatistic
175175
state.get(key)
176176
}
177177

178-
fn put(&self, key: &TableScopedPath, value: CachedFileMetadata) -> Option<CachedFileMetadata> {
178+
fn put(
179+
&self,
180+
key: &TableScopedPath,
181+
value: CachedFileMetadata,
182+
) -> Option<CachedFileMetadata> {
179183
let mut state = self.state.lock().unwrap();
180184
state.put(key, value)
181185
}
@@ -238,7 +242,10 @@ impl FileStatisticsCache for DefaultFileStatisticsCache {
238242
entries
239243
}
240244

241-
fn drop_table_entries(&self, table_ref: &Option<TableReference>) -> datafusion_common::Result<()> {
245+
fn drop_table_entries(
246+
&self,
247+
table_ref: &Option<TableReference>,
248+
) -> datafusion_common::Result<()> {
242249
let mut state = self.state.lock().unwrap();
243250
let mut table_paths = vec![];
244251
for (path, _) in state.lru_queue.list_entries() {
@@ -294,7 +301,7 @@ mod tests {
294301
false,
295302
)]);
296303

297-
let path = TableScopedPath{
304+
let path = TableScopedPath {
298305
path: meta.location.clone(),
299306
table: None,
300307
};
@@ -313,14 +320,14 @@ mod tests {
313320
// Cache hit
314321
let result = cache.get(&path);
315322
assert!(result.is_some());
323+
316324
let cached = result.unwrap();
317325
assert!(cached.is_valid_for(&meta));
318326

319-
320327
// File size changed - validation should fail
321328
let meta2 = create_test_meta("test", 2048);
322329

323-
let path_2 = TableScopedPath{
330+
let path_2 = TableScopedPath {
324331
path: meta2.location.clone(),
325332
table: None,
326333
};
@@ -340,7 +347,7 @@ mod tests {
340347
let entries = cache.list_entries();
341348
assert_eq!(entries.len(), 1);
342349

343-
let path_3 = TableScopedPath{
350+
let path_3 = TableScopedPath {
344351
path: Path::from("test"),
345352
table: None,
346353
};
@@ -446,7 +453,10 @@ mod tests {
446453
#[test]
447454
fn test_cache_invalidation_on_file_modification() {
448455
let cache = DefaultFileStatisticsCache::default();
449-
let path = TableScopedPath { table: None, path : Path::from("test.parquet"), };
456+
let path = TableScopedPath {
457+
path: Path::from("test.parquet"),
458+
table: None,
459+
};
450460
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
451461

452462
let meta_v1 = create_test_meta("test.parquet", 100);
@@ -482,7 +492,10 @@ mod tests {
482492
#[test]
483493
fn test_ordering_cache_invalidation_on_file_modification() {
484494
let cache = DefaultFileStatisticsCache::default();
485-
let path = TableScopedPath { path: Path::from("test.parquet"), table: None };
495+
let path = TableScopedPath {
496+
path: Path::from("test.parquet"),
497+
table: None,
498+
};
486499
let schema = Schema::new(vec![Field::new("a", DataType::Int32, false)]);
487500

488501
// Cache with original metadata and ordering
@@ -554,7 +567,10 @@ mod tests {
554567
None,
555568
);
556569

557-
let path_1 = TableScopedPath { path: meta1.location.clone(), table: None };
570+
let path_1 = TableScopedPath {
571+
path: meta1.location.clone(),
572+
table: None,
573+
};
558574

559575
cache.put(&path_1, cached_value);
560576
let meta2 = create_test_meta("test2.parquet", 200);
@@ -564,7 +580,10 @@ mod tests {
564580
Some(ordering()),
565581
);
566582

567-
let path_2 = TableScopedPath { path: meta2.location.clone(), table: None };
583+
let path_2 = TableScopedPath {
584+
path: meta2.location.clone(),
585+
table: None,
586+
};
568587

569588
cache.put(&path_2, cached_value);
570589

@@ -611,8 +630,16 @@ mod tests {
611630

612631
// create a cache with a limit which fits exactly 2 entries
613632
let cache = DefaultFileStatisticsCache::new(limit_for_2_entries);
614-
let path_1 = TableScopedPath { path: meta_1.location.clone(), table: None };
615-
let path_2 = TableScopedPath { path: meta_2.location.clone(), table: None };
633+
let path_1 = TableScopedPath {
634+
path: meta_1.location.clone(),
635+
table: None,
636+
};
637+
638+
let path_2 = TableScopedPath {
639+
path: meta_2.location.clone(),
640+
table: None,
641+
};
642+
616643
cache.put(&path_1, value_1.clone());
617644
cache.put(&path_2, value_2.clone());
618645

@@ -624,8 +651,10 @@ mod tests {
624651
assert_eq!(result_1.unwrap(), value_1);
625652
assert_eq!(result_2.unwrap(), value_2);
626653

627-
let path_3 = TableScopedPath { path: meta_3.location.clone(), table: None };
628-
654+
let path_3 = TableScopedPath {
655+
path: meta_3.location.clone(),
656+
table: None,
657+
};
629658

630659
// adding the third entry evicts the first entry
631660
cache.put(&path_3, value_3.clone());
@@ -662,7 +691,10 @@ mod tests {
662691
// create a cache with a size less than the entry
663692
let cache = DefaultFileStatisticsCache::new(limit_less_than_the_entry);
664693

665-
let path_1 = TableScopedPath { path: meta.location.clone(), table: None };
694+
let path_1 = TableScopedPath {
695+
path: meta.location.clone(),
696+
table: None,
697+
};
666698

667699
cache.put(&path_1, value);
668700

0 commit comments

Comments
 (0)