set encoding to support Windows

Author: yqzhishen

data_gen/singing/binarize.py

@@ -59,8 +59,8 @@ def load_meta_data(self):
                 item_name = raw_item_name = piece_path[len(processed_data_dir)+1:].replace('/', '-')[:-len(wav_suffix)]
                 if len(self.processed_data_dirs) > 1:
                     item_name = f'ds{ds_id}_{item_name}'
-                self.item2txt[item_name] = open(f'{piece_path.replace(wav_suffix, txt_suffix)}').readline()
-                self.item2ph[item_name] = open(f'{piece_path.replace(wav_suffix, ph_suffix)}').readline()
+                self.item2txt[item_name] = open(f'{piece_path.replace(wav_suffix, txt_suffix)}', encoding='utf-8').readline()
+                self.item2ph[item_name] = open(f'{piece_path.replace(wav_suffix, ph_suffix)}', encoding='utf-8').readline()
                 self.item2wavfn[item_name] = piece_path
 
                 self.item2spk[item_name] = re.split('-|#', piece_path.split('/')[-2])[0]
@@ -106,10 +106,10 @@ def _phone_encoder(self):
             for ph_sent in self.item2ph.values():
                 ph_set += ph_sent.split(' ')
             ph_set = sorted(set(ph_set))
-            json.dump(ph_set, open(ph_set_fn, 'w'))
+            json.dump(ph_set, open(ph_set_fn, 'w', encoding='utf-8'))
             print("| Build phone set: ", ph_set)
         else:
-            ph_set = json.load(open(ph_set_fn, 'r'))
+            ph_set = json.load(open(ph_set_fn, 'r', encoding='utf-8'))
             print("| Load phone set: ", ph_set)
         return build_phone_encoder(hparams['binary_data_dir'])
 
@@ -189,7 +189,7 @@ class MidiSingingBinarizer(SingingBinarizer):
 
     def load_meta_data(self):
         for ds_id, processed_data_dir in enumerate(self.processed_data_dirs):
-            meta_midi = json.load(open(os.path.join(processed_data_dir, 'meta.json')))   # [list of dict]
+            meta_midi = json.load(open(os.path.join(processed_data_dir, 'meta.json'), encoding='utf-8'))   # [list of dict]
 
             for song_item in meta_midi:
                 item_name = raw_item_name = song_item['item_name']
@@ -303,7 +303,7 @@ def split_train_test_set(self, item_names):
     def load_meta_data(self):
         raw_data_dir = hparams['raw_data_dir']
         # meta_midi = json.load(open(os.path.join(raw_data_dir, 'meta.json')))   # [list of dict]
-        utterance_labels = open(os.path.join(raw_data_dir, 'transcriptions.txt')).readlines()
+        utterance_labels = open(os.path.join(raw_data_dir, 'transcriptions.txt'), encoding='utf-8').readlines()
 
         for utterance_label in utterance_labels:
             song_info = utterance_label.split('|')

data_gen/tts/base_binarizer.py

@@ -85,11 +85,11 @@ def _phone_encoder(self):
         ph_set = []
         if hparams['reset_phone_dict'] or not os.path.exists(ph_set_fn):
             for processed_data_dir in self.processed_data_dirs:
-                ph_set += [x.split(' ')[0] for x in open(f'{processed_data_dir}/dict.txt').readlines()]
+                ph_set += [x.split(' ')[0] for x in open(f'{processed_data_dir}/dict.txt', encoding='utf-8').readlines()]
             ph_set = sorted(set(ph_set))
-            json.dump(ph_set, open(ph_set_fn, 'w'))
+            json.dump(ph_set, open(ph_set_fn, 'w', encoding='utf-8'))
         else:
-            ph_set = json.load(open(ph_set_fn, 'r'))
+            ph_set = json.load(open(ph_set_fn, 'r', encoding='utf-8'))
         print("| phone set: ", ph_set)
         return build_phone_encoder(hparams['binary_data_dir'])
 
@@ -113,7 +113,7 @@ def process(self):
         self.spk_map = self.build_spk_map()
         print("| spk_map: ", self.spk_map)
         spk_map_fn = f"{hparams['binary_data_dir']}/spk_map.json"
-        json.dump(self.spk_map, open(spk_map_fn, 'w'))
+        json.dump(self.spk_map, open(spk_map_fn, 'w', encoding='utf-8'))
 
         self.phone_encoder = self._phone_encoder()
         self.process_data('valid')

data_gen/tts/data_gen_utils.py

@@ -273,7 +273,7 @@ def toJson(self):
 
 def get_mel2ph(tg_fn, ph, mel, hparams):
     ph_list = ph.split(" ")
-    with open(tg_fn, "r") as f:
+    with open(tg_fn, "r", encoding='utf-8') as f:
         tg = f.readlines()
     tg = remove_empty_lines(tg)
     tg = TextGrid(tg)
@@ -339,7 +339,7 @@ def get_mel2ph(tg_fn, ph, mel, hparams):
 
 def build_phone_encoder(data_dir):
     phone_list_file = os.path.join(data_dir, 'phone_set.json')
-    phone_list = json.load(open(phone_list_file))
+    phone_list = json.load(open(phone_list_file, encoding='utf-8'))
     return TokenTextEncoder(None, vocab_list=phone_list, replace_oov=',')
 
 

inference/svs/opencpop/map.py

@@ -1,7 +1,7 @@
 def cpop_pinyin2ph_func():
     # In the README file of opencpop dataset, they defined a "pinyin to phoneme mapping table"
     pinyin2phs = {'AP': 'AP', 'SP': 'SP'}
-    with open('inference/svs/opencpop/cpop_pinyin2ph.txt') as rf:
+    with open('inference/svs/opencpop/cpop_pinyin2ph.txt', encoding='utf-8') as rf:
         for line in rf.readlines():
             elements = [x.strip() for x in line.split('|') if x.strip() != '']
             pinyin2phs[elements[0]] = elements[1]

modules/parallel_wavegan/utils/utils.py

@@ -137,7 +137,7 @@ def __init__(self, feats_scp, default_hdf5_path="feats"):
 
         """
         self.default_hdf5_path = default_hdf5_path
-        with open(feats_scp) as f:
+        with open(feats_scp, encoding='utf-8') as f:
             lines = [line.replace("\n", "") for line in f.readlines()]
         self.data = {}
         for line in lines:

tasks/tts/tts.py

@@ -95,7 +95,7 @@ def shuffle_batches(batches):
     def build_phone_encoder(self, data_dir):
         phone_list_file = os.path.join(data_dir, 'phone_set.json')
 
-        phone_list = json.load(open(phone_list_file))
+        phone_list = json.load(open(phone_list_file, encoding='utf-8'))
         return TokenTextEncoder(None, vocab_list=phone_list, replace_oov=',')
 
     def build_optimizer(self, model):

utils/hparams.py

@@ -96,7 +96,7 @@ def load_config(config_fn):  # deep first
 
     if args_work_dir != '' and (not os.path.exists(ckpt_config_path) or args.reset) and not args.infer:
         os.makedirs(hparams_['work_dir'], exist_ok=True)
-        with open(ckpt_config_path, 'w') as f:
+        with open(ckpt_config_path, 'w', encoding='utf-8') as f:
             yaml.safe_dump(hparams_, f)
 
     hparams_['infer'] = args.infer

vocoders/hifigan.py

@@ -21,7 +21,7 @@ def load_model(config_path, checkpoint_path):
         config = set_hparams(config_path, global_hparams=False)
         state = ckpt_dict["state_dict"]["model_gen"]
     elif '.json' in config_path:
-        config = json.load(open(config_path, 'r'))
+        config = json.load(open(config_path, 'r', encoding='utf-8'))
         state = ckpt_dict["generator"]
 
     model = HifiGanGenerator(config)