-
Notifications
You must be signed in to change notification settings - Fork 136
/
22d24867-eb5a-4fcc-ae2c-263d0277dfd1.txt
5740 lines (5673 loc) · 383 KB
/
22d24867-eb5a-4fcc-ae2c-263d0277dfd1.txt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
====================================================================================================
import os
import sys
with open(sys.argv[0]) as f:
code = f.read() # read the code of this file ASAP, for logging
import uuid
import glob
import time
from dataclasses import dataclass
import numpy as np
import torch
from torch import nn
import torch.nn.functional as F
import torch.distributed as dist
import torch._inductor.config as config
from torch.nn.parallel import DistributedDataParallel as DDP
# -----------------------------------------------------------------------------
# Muon optimizer
def zeropower_via_svd(G, steps=None):
U, S, V = G.svd()
return U @ V.T
@torch.compile
def zeropower_via_newtonschulz5(G, steps=10, eps=1e-7):
"""
Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
zero even beyond the point where the iteration no longer converges all the way to one everywhere
on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
where S' is diagonal with S_{ii}' \sim Uniform(0.5, 1.5), which turns out not to hurt model
performance at all relative to UV^T, where USV^T = G is the SVD.
"""
assert len(G.shape) == 2
a, b, c = (3.4445, -4.7750, 2.0315)
X = G.bfloat16()
X /= (X.norm() + eps) # ensure top singular value <= 1
if G.size(0) > G.size(1):
X = X.T
for _ in range(steps):
A = X @ X.T
B = A @ X
X = a * X + b * B + c * A @ B
if G.size(0) > G.size(1):
X = X.T
return X
zeropower_backends = dict(svd=zeropower_via_svd, newtonschulz5=zeropower_via_newtonschulz5)
class Muon(torch.optim.Optimizer):
"""
Muon - MomentUm Orthogonalized by Newton-schulz
Muon internally runs standard SGD-momentum, and then performs an orthogonalization post-
processing step, in which each 2D parameter's update is replaced with the nearest orthogonal
matrix. To efficiently orthogonalize each update, we use a Newton-Schulz iteration, which has
the advantage that it can be stably run in bfloat16 on the GPU.
Some warnings:
- This optimizer assumes that all parameters passed in are 2D.
- It should not be used for the embedding layer, the final fully connected layer, or any {0,1}-D
parameters; those should all be optimized by a standard method (e.g., AdamW).
- To use it with 4D convolutional filters, it works well to just flatten their last 3 dimensions.
- We believe it is unlikely to work well for training with small batch size.
- We believe it may not work well for finetuning pretrained models, but we haven't tested this.
- We have not yet tried this optimizer for training scenarios larger than NanoGPT (124M).
Arguments:
lr: The learning rate used by the internal SGD.
momentum: The momentum used by the internal SGD.
nesterov: Whether to use Nesterov-style momentum in the internal SGD. (recommended)
backend: The chosen backend for the orthogonalization step. (recommended: 'newtonschulz5')
backend_steps: The number of iteration steps to use in the backend, if it is iterative.
"""
def __init__(self, params, lr=3e-4, momentum=0.95, nesterov=True,
backend='newtonschulz5', backend_steps=5,
rank=0, world_size=1):
defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, backend=backend, backend_steps=backend_steps)
super().__init__(params, defaults)
self.rank = rank
self.world_size = world_size
def step(self):
for group in self.param_groups:
lr = group['lr']
momentum = group['momentum']
zeropower_backend = zeropower_backends[group['backend']]
# generate weight updates in distributed fashion
total_params = sum(p.numel() for p in group['params'])
updates_flat = torch.zeros(total_params, device='cuda', dtype=torch.bfloat16)
curr_idx = 0
for i, p in enumerate(group['params']):
# luckily this will perfectly distribute a transformer with multiple of 4 layers to 8 GPUs
if i % self.world_size == self.rank:
g = p.grad
if g is None:
continue
state = self.state[p]
if 'momentum_buffer' not in state:
state['momentum_buffer'] = torch.zeros_like(g)
buf = state['momentum_buffer']
buf.mul_(momentum).add_(g)
if group['nesterov']:
g = g.add(buf, alpha=momentum)
g = zeropower_backend(g, steps=group['backend_steps'])
g *= max(g.size(0), g.size(1))**0.5 # scale to have update.square().mean() == 1
updates_flat[curr_idx:curr_idx+p.numel()] = g.flatten()
curr_idx += p.numel()
# sync updates across devices. we are not memory-constrained so can do this simple deserialization
dist.all_reduce(updates_flat, op=dist.ReduceOp.SUM)
# deserialize and apply updates
curr_idx = 0
for p in group['params']:
g = updates_flat[curr_idx:curr_idx+p.numel()].view_as(p.data).type_as(p.data)
p.data.add_(g, alpha=-lr)
curr_idx += p.numel()
# -----------------------------------------------------------------------------
# PyTorch nn.Module definitions for the GPT-2 model
class Rotary(torch.nn.Module):
def __init__(self, dim, base=10000):
super().__init__()
self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / dim))
self.seq_len_cached = None
self.cos_cached = None
self.sin_cached = None
def forward(self, x):
seq_len = x.shape[1]
if seq_len != self.seq_len_cached:
self.seq_len_cached = seq_len
t = torch.arange(seq_len, device=x.device).type_as(self.inv_freq)
freqs = torch.outer(t, self.inv_freq).to(x.device)
self.cos_cached = freqs.cos().bfloat16()
self.sin_cached = freqs.sin().bfloat16()
return self.cos_cached[None, :, None, :], self.sin_cached[None, :, None, :]
def apply_rotary_emb(x, cos, sin):
assert x.ndim == 4 # multihead attention
d = x.shape[3]//2
x1 = x[..., :d]
x2 = x[..., d:]
y1 = x1 * cos + x2 * sin
y2 = x1 * (-sin) + x2 * cos
return torch.cat([y1, y2], 3).type_as(x)
class CausalSelfAttention(nn.Module):
def __init__(self, config):
super().__init__()
self.n_head = config.n_head
self.n_embd = config.n_embd
self.head_dim = self.n_embd // self.n_head
assert self.n_embd % self.n_head == 0
self.c_q = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_k = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_v = nn.Linear(self.n_embd, self.n_embd, bias=False)
# output projection
self.c_proj = nn.Linear(self.n_embd, self.n_embd, bias=False)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
self.rotary = Rotary(self.head_dim)
def forward(self, x):
B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd)
q = self.c_q(x).view(B, T, self.n_head, self.head_dim)
k = self.c_k(x).view(B, T, self.n_head, self.head_dim)
v = self.c_v(x).view(B, T, self.n_head, self.head_dim)
cos, sin = self.rotary(q)
q, k = F.rms_norm(q, (q.size(-1),)), F.rms_norm(k, (k.size(-1),)) # QK norm suggested by @Grad62304977
q, k = apply_rotary_emb(q, cos, sin), apply_rotary_emb(k, cos, sin)
y = F.scaled_dot_product_attention(q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), is_causal=True)
y = y.transpose(1, 2).contiguous().view_as(x) # re-assemble all head outputs side by side
y = self.c_proj(y)
return y
class MLP(nn.Module):
def __init__(self, config):
super().__init__()
self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=False)
self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=False)
self.c_proj.weight.data.zero_() # zero init suggested by @Grad62304977
def forward(self, x):
x = self.c_fc(x)
x = F.relu(x).square() # https://arxiv.org/abs/2109.08668v2; ~1-2% better than GELU; suggested by @SKYLINEZ007 and @Grad62304977
x = self.c_proj(x)
return x
class Block(nn.Module):
def __init__(self, config):
super().__init__()
self.attn = CausalSelfAttention(config)
self.mlp = MLP(config)
def forward(self, x):
x = x + self.attn(F.rms_norm(x, (x.size(-1),)))
x = x + self.mlp(F.rms_norm(x, (x.size(-1),)))
return x
# -----------------------------------------------------------------------------
# The main GPT-2 model
@dataclass
class GPTConfig:
vocab_size : int = 50304
n_layer : int = 12
n_head : int = 6 # head dim 128 suggested by @Grad62304977
n_embd : int = 768
class GPT(nn.Module):
def __init__(self, config):
super().__init__()
self.config = config
self.transformer = nn.ModuleDict(dict(
wte = nn.Embedding(config.vocab_size, config.n_embd),
h = nn.ModuleList([Block(config) for _ in range(config.n_layer)]),
))
self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
self.transformer.wte.weight = self.lm_head.weight # https://paperswithcode.com/method/weight-tying
def forward(self, idx, targets=None, return_logits=True):
# forward the GPT model itself
x = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd)
for block in self.transformer.h:
x = block(x)
x = F.rms_norm(x, (x.size(-1),))
if targets is not None:
# if we are given some desired targets also calculate the loss
logits = self.lm_head(x)
logits = logits.float() # use tf32/fp32 for logits
loss = F.cross_entropy(logits.view(-1, logits.size(-1)), targets.view(-1), ignore_index=-1)
else:
# inference-time mini-optimization: only forward the lm_head on the very last position
logits = self.lm_head(x[:, [-1], :]) # note: using list [-1] to preserve the time dim
logits = logits.float() # use tf32/fp32 for logits
loss = None
# there are performance reasons why not returning logits is prudent, if not needed
if not return_logits:
logits = None
return logits, loss
# -----------------------------------------------------------------------------
# Our own simple Distributed Data Loader
def _peek_data_shard(filename):
# only reads the header, returns header data
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
if header[0] != 20240520:
print("ERROR: magic number mismatch in the data .bin file!")
print("---> HINT: Are you passing in a correct file with --input_bin?")
print("---> HINT: Dataset encoding changed recently, re-run data prepro or refer again to README")
print("---> HINT: For example re-run: `python dev/data/tinyshakespeare.py`, then re-try")
exit(1)
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
return ntok # for now just return the number of tokens
def _load_data_shard(filename):
with open(filename, "rb") as f:
# first read the header, which is 256 int32 integers (4 bytes each)
header = np.frombuffer(f.read(256*4), dtype=np.int32)
assert header[0] == 20240520, "magic number mismatch in the data .bin file"
assert header[1] == 1, "unsupported version"
ntok = header[2] # number of tokens (claimed)
# the rest of it are tokens, stored as uint16
tokens = np.frombuffer(f.read(), dtype=np.uint16)
assert len(tokens) == ntok, "number of tokens read does not match header?"
return tokens
class DistributedDataLoader:
def __init__(self, filename_pattern, B, T, process_rank, num_processes):
self.process_rank = process_rank
self.num_processes = num_processes
self.B = B
self.T = T
# glob files that match the pattern
self.files = sorted(glob.glob(filename_pattern))
assert len(self.files) > 0, f"did not find any files that match the pattern {filename_pattern}"
# load and validate all data shards, count number of tokens in total
ntok_total = 0
for fname in self.files:
shard_ntok = _peek_data_shard(fname)
assert shard_ntok >= num_processes * B * T + 1
ntok_total += int(shard_ntok)
self.ntok_total = ntok_total
# kick things off
self.reset()
def reset(self):
self.current_shard = 0
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def advance(self): # advance to next data shard
self.current_shard = (self.current_shard + 1) % len(self.files)
self.current_position = self.process_rank * self.B * self.T
self.tokens = _load_data_shard(self.files[self.current_shard])
def next_batch(self):
B = self.B
T = self.T
buf = self.tokens[self.current_position : self.current_position+B*T+1]
buf = torch.tensor(buf.astype(np.int32), dtype=torch.long)
x = (buf[:-1]).view(B, T) # inputs
y = (buf[1:]).view(B, T) # targets
# advance current position and load next shard if necessary
self.current_position += B * T * self.num_processes
if self.current_position + (B * T * self.num_processes + 1) > len(self.tokens):
self.advance()
return x.cuda(), y.cuda()
# -----------------------------------------------------------------------------
# int main
@dataclass
class Hyperparameters:
# data hyperparams
input_bin : str = 'data/fineweb10B/fineweb_train_*.bin' # input .bin to train on
input_val_bin : str = 'data/fineweb10B/fineweb_val_*.bin' # input .bin to eval validation loss on
# optimization hyperparams
batch_size : int = 8*64 # batch size, in sequences, across all devices
device_batch_size : int = 64 # batch size, in sequences, per device
sequence_length : int = 1024 # sequence length, in tokens
num_iterations : int = 5100 # number of iterations to run
learning_rate : float = 0.0036
warmup_iters : int = 0
warmdown_iters : int = 1450 # number of iterations of linear warmup/warmdown for triangular or trapezoidal schedule
weight_decay : float = 0
# evaluation and logging hyperparams
val_loss_every : int = 125 # every how many steps to evaluate val loss? 0 for only at the end
val_tokens : int = 10485760 # how many tokens of validation data? it's important to keep this fixed for consistent comparisons
save_every : int = 0 # every how many steps to save the checkpoint? 0 for only at the end
args = Hyperparameters()
# set up DDP (distributed data parallel). torchrun sets this env variable
assert torch.cuda.is_available()
dist.init_process_group(backend='nccl')
ddp_rank = int(os.environ['RANK'])
ddp_local_rank = int(os.environ['LOCAL_RANK'])
ddp_world_size = int(os.environ['WORLD_SIZE'])
device = f'cuda:{ddp_local_rank}'
torch.cuda.set_device(device)
print(f"using device: {device}")
master_process = (ddp_rank == 0) # this process will do logging, checkpointing etc.
# convenience variables
B, T = args.device_batch_size, args.sequence_length
# calculate the number of steps to take in the val loop.
assert args.val_tokens % (B * T * ddp_world_size) == 0
val_steps = args.val_tokens // (B * T * ddp_world_size)
# calculate the steps of gradient accumulation required to attain the desired global batch size.
assert args.batch_size % (B * ddp_world_size) == 0
train_accumulation_steps = args.batch_size // (B * ddp_world_size)
# load tokens
train_loader = DistributedDataLoader(args.input_bin, B, T, ddp_rank, ddp_world_size)
val_loader = DistributedDataLoader(args.input_val_bin, B, T, ddp_rank, ddp_world_size)
if master_process:
print(f"Training DataLoader: total number of tokens: {train_loader.ntok_total} across {len(train_loader.files)} files")
print(f"Validation DataLoader: total number of tokens: {val_loader.ntok_total} across {len(val_loader.files)} files")
x, y = train_loader.next_batch()
# there are only 50257 unique GPT-2 tokens; we extend to nearest multiple of 128 for efficiency. suggested to me by @Grad62304977.
# this originates from Karpathy's experiments.
num_vocab = 50304
model = GPT(GPTConfig(vocab_size=num_vocab, n_layer=12, n_head=6, n_embd=768))
model = model.cuda()
if hasattr(config, "coordinate_descent_tuning"):
config.coordinate_descent_tuning = True # suggested by @Chillee
model = torch.compile(model)
# here we wrap model into DDP container
model = DDP(model, device_ids=[ddp_local_rank])
raw_model = model.module # always contains the "raw" unwrapped model
ctx = torch.amp.autocast(device_type='cuda', dtype=torch.bfloat16)
# init the optimizer(s)
optimizer1 = torch.optim.AdamW(raw_model.lm_head.parameters(), lr=args.learning_rate, betas=(0.9, 0.95),
weight_decay=args.weight_decay, fused=True)
optimizer2 = Muon(raw_model.transformer.h.parameters(), lr=0.1*args.learning_rate, momentum=0.95,
rank=ddp_rank, world_size=ddp_world_size)
optimizers = [optimizer1, optimizer2]
# learning rate decay scheduler (linear warmup and warmdown)
def get_lr(it):
assert it <= args.num_iterations
# 1) linear warmup for warmup_iters steps
if it < args.warmup_iters:
return (it+1) / args.warmup_iters
# 2) constant lr for a while
elif it < args.num_iterations - args.warmdown_iters:
return 1.0
# 3) linear warmdown
else:
decay_ratio = (args.num_iterations - it) / args.warmdown_iters
return decay_ratio
schedulers = [torch.optim.lr_scheduler.LambdaLR(opt, get_lr) for opt in optimizers]
# begin logging
if master_process:
run_id = str(uuid.uuid4())
logdir = 'logs/%s/' % run_id
os.makedirs(logdir, exist_ok=True)
logfile = 'logs/%s.txt' % run_id
# create the log file
with open(logfile, "w") as f:
# begin the log by printing this file (the Python code)
f.write('='*100 + '\n')
f.write(code)
f.write('='*100 + '\n')
# log information about the hardware/software environment this is running on
# and print the full `nvidia-smi` to file
f.write(f"Running pytorch {torch.version.__version__} compiled for CUDA {torch.version.cuda}\nnvidia-smi:\n")
import subprocess
result = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
f.write(f'{result.stdout}\n')
f.write('='*100 + '\n')
training_time_ms = 0
# start the clock
torch.cuda.synchronize()
t0 = time.time()
# begin training
train_loader.reset()
for step in range(args.num_iterations + 1):
last_step = (step == args.num_iterations)
# This effectively ignores timing first 10 steps, which are slower for weird reasons.
# Alternately, and slightly more correctly in terms of benchmarking, we could do 10
# steps with dummy data first, and then re-initialize the model and reset the loader.
if step == 10:
training_time_ms = 0
t0 = time.time()
timed_steps = float('nan') if step <= 11 else (step - 10) + 1 # <= 11 to avoid bug in val
# once in a while evaluate the validation dataset
if (last_step or (args.val_loss_every > 0 and step % args.val_loss_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# run validation batches
model.eval()
val_loader.reset()
val_loss = 0.0
for _ in range(val_steps):
x_val, y_val = val_loader.next_batch()
with ctx: # of course, we'd like to use no_grad() here too, but that creates a torch.compile error for some reason
_, loss = model(x_val, y_val, return_logits=False)
val_loss += loss.detach()
del loss
dist.all_reduce(val_loss, op=dist.ReduceOp.AVG)
val_loss /= val_steps
# log val loss to console and to logfile
if master_process:
print(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms')
with open(logfile, "a") as f:
f.write(f'step:{step}/{args.num_iterations} val_loss:{val_loss:.4f} train_time:{training_time_ms:.0f}ms step_avg:{training_time_ms/(timed_steps-1):.2f}ms\n')
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
if master_process and (last_step or (args.save_every > 0 and step % args.save_every == 0)):
# stop the clock
torch.cuda.synchronize()
training_time_ms += 1000 * (time.time() - t0)
# save the state of the training process
log = dict(step=step, code=code, model=raw_model.state_dict(), optimizers=[opt.state_dict() for opt in optimizers])
torch.save(log, 'logs/%s/state_step%06d.pt' % (run_id, step))
# start the clock again
torch.cuda.synchronize()
t0 = time.time()
# bit confusing: we want to make sure to eval on 0th iteration
# but also after the very last iteration. so we loop for step <= num_iterations
# instead of just < num_iterations (one extra due to <=), only to do
# the validation/sampling one last time, and then we break right here as we're done.
if last_step:
break
# --------------- TRAINING SECTION BEGIN -----------------
model.train()
for i in range(1, train_accumulation_steps+1):
# forward pass
with ctx:
_, loss = model(x, y, return_logits=False)
train_loss = loss.detach()
# advance the dataset for the next batch
x, y = train_loader.next_batch()
# backward pass
if i < train_accumulation_steps:
with model.no_sync(): # there's no need to sync gradients every accumulation step
loss.backward()
else:
loss.backward() # just sync on the last step
for p in model.parameters():
p.grad /= train_accumulation_steps
# step the optimizers and schedulers
for opt, sched in zip(optimizers, schedulers):
opt.step()
sched.step()
# null the gradients
model.zero_grad(set_to_none=True)
# --------------- TRAINING SECTION END -------------------
# everything that follows now is just diagnostics, prints, logging, etc.
#dist.all_reduce(train_loss, op=dist.ReduceOp.AVG) # all-reducing the training loss would be more correct in terms of logging, but slower
if master_process:
approx_time = training_time_ms + 1000 * (time.time() - t0)
print(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms")
with open(logfile, "a") as f:
f.write(f"step:{step+1}/{args.num_iterations} train_loss:{train_loss.item():.4f} train_time:{approx_time:.0f}ms step_avg:{approx_time/timed_steps:.2f}ms\n")
if master_process:
print(f"peak memory consumption: {torch.cuda.max_memory_allocated() // 1024 // 1024} MiB")
# -------------------------------------------------------------------------
# clean up nice
dist.destroy_process_group()
====================================================================================================
Running pytorch 2.4.1+cu121 compiled for CUDA 12.1
nvidia-smi:
Wed Oct 16 13:25:45 2024
+-----------------------------------------------------------------------------------------+
| NVIDIA-SMI 555.42.06 Driver Version: 555.42.06 CUDA Version: 12.5 |
|-----------------------------------------+------------------------+----------------------+
| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |
| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |
| | | MIG M. |
|=========================================+========================+======================|
| 0 NVIDIA H100 80GB HBM3 Off | 00000000:18:00.0 Off | 0 |
| N/A 28C P0 111W / 700W | 5786MiB / 81559MiB | 9% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 1 NVIDIA H100 80GB HBM3 Off | 00000000:2A:00.0 Off | 0 |
| N/A 30C P0 116W / 700W | 5834MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 2 NVIDIA H100 80GB HBM3 Off | 00000000:3A:00.0 Off | 0 |
| N/A 31C P0 113W / 700W | 5834MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 3 NVIDIA H100 80GB HBM3 Off | 00000000:5D:00.0 Off | 0 |
| N/A 28C P0 115W / 700W | 5834MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 4 NVIDIA H100 80GB HBM3 Off | 00000000:84:00.0 Off | 0 |
| N/A 28C P0 113W / 700W | 5834MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 5 NVIDIA H100 80GB HBM3 Off | 00000000:8B:00.0 Off | 0 |
| N/A 30C P0 115W / 700W | 5834MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 6 NVIDIA H100 80GB HBM3 Off | 00000000:91:00.0 Off | 0 |
| N/A 29C P0 112W / 700W | 5834MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
| 7 NVIDIA H100 80GB HBM3 Off | 00000000:E4:00.0 Off | 0 |
| N/A 28C P0 116W / 700W | 5594MiB / 81559MiB | 0% Default |
| | | Disabled |
+-----------------------------------------+------------------------+----------------------+
+-----------------------------------------------------------------------------------------+
| Processes: |
| GPU GI CI PID Type Process name GPU Memory |
| ID ID Usage |
|=========================================================================================|
| 0 N/A N/A 20785 C /usr/bin/python3 0MiB |
| 1 N/A N/A 20786 C /usr/bin/python3 0MiB |
| 2 N/A N/A 20787 C /usr/bin/python3 0MiB |
| 3 N/A N/A 20788 C /usr/bin/python3 0MiB |
| 4 N/A N/A 20789 C /usr/bin/python3 0MiB |
| 5 N/A N/A 20790 C /usr/bin/python3 0MiB |
| 6 N/A N/A 20791 C /usr/bin/python3 0MiB |
| 7 N/A N/A 20792 C /usr/bin/python3 0MiB |
+-----------------------------------------------------------------------------------------+
====================================================================================================
step:0/5100 val_loss:16.0098 train_time:286ms step_avg:nanms
step:1/5100 train_loss:16.0051 train_time:60444ms step_avg:nanms
step:2/5100 train_loss:9.5200 train_time:60535ms step_avg:nanms
step:3/5100 train_loss:8.7468 train_time:60688ms step_avg:nanms
step:4/5100 train_loss:8.0303 train_time:60841ms step_avg:nanms
step:5/5100 train_loss:7.4920 train_time:60993ms step_avg:nanms
step:6/5100 train_loss:7.4504 train_time:61145ms step_avg:nanms
step:7/5100 train_loss:7.3315 train_time:61298ms step_avg:nanms
step:8/5100 train_loss:7.5719 train_time:61450ms step_avg:nanms
step:9/5100 train_loss:7.4521 train_time:61604ms step_avg:nanms
step:10/5100 train_loss:7.0940 train_time:61759ms step_avg:nanms
step:11/5100 train_loss:6.9980 train_time:90ms step_avg:nanms
step:12/5100 train_loss:6.8961 train_time:242ms step_avg:nanms
step:13/5100 train_loss:6.7010 train_time:395ms step_avg:131.59ms
step:14/5100 train_loss:6.6766 train_time:548ms step_avg:136.89ms
step:15/5100 train_loss:6.6384 train_time:700ms step_avg:140.03ms
step:16/5100 train_loss:6.5587 train_time:853ms step_avg:142.19ms
step:17/5100 train_loss:6.5682 train_time:1009ms step_avg:144.12ms
step:18/5100 train_loss:6.5854 train_time:1163ms step_avg:145.32ms
step:19/5100 train_loss:6.4148 train_time:1315ms step_avg:146.14ms
step:20/5100 train_loss:6.4305 train_time:1469ms step_avg:146.92ms
step:21/5100 train_loss:6.0939 train_time:1622ms step_avg:147.48ms
step:22/5100 train_loss:6.4600 train_time:1777ms step_avg:148.05ms
step:23/5100 train_loss:6.6648 train_time:1929ms step_avg:148.36ms
step:24/5100 train_loss:6.3462 train_time:2082ms step_avg:148.74ms
step:25/5100 train_loss:6.4746 train_time:2236ms step_avg:149.06ms
step:26/5100 train_loss:6.1830 train_time:2388ms step_avg:149.27ms
step:27/5100 train_loss:6.1035 train_time:2542ms step_avg:149.55ms
step:28/5100 train_loss:6.2450 train_time:2694ms step_avg:149.68ms
step:29/5100 train_loss:5.9279 train_time:2848ms step_avg:149.89ms
step:30/5100 train_loss:6.2054 train_time:3001ms step_avg:150.03ms
step:31/5100 train_loss:6.0422 train_time:3154ms step_avg:150.20ms
step:32/5100 train_loss:6.0101 train_time:3308ms step_avg:150.35ms
step:33/5100 train_loss:5.8360 train_time:3461ms step_avg:150.48ms
step:34/5100 train_loss:6.1218 train_time:3613ms step_avg:150.56ms
step:35/5100 train_loss:6.0536 train_time:3766ms step_avg:150.66ms
step:36/5100 train_loss:6.1885 train_time:3919ms step_avg:150.74ms
step:37/5100 train_loss:6.1319 train_time:4073ms step_avg:150.86ms
step:38/5100 train_loss:6.0204 train_time:4226ms step_avg:150.91ms
step:39/5100 train_loss:5.9126 train_time:4381ms step_avg:151.06ms
step:40/5100 train_loss:5.9277 train_time:4533ms step_avg:151.11ms
step:41/5100 train_loss:5.8419 train_time:4686ms step_avg:151.15ms
step:42/5100 train_loss:5.8712 train_time:4839ms step_avg:151.20ms
step:43/5100 train_loss:5.7509 train_time:4992ms step_avg:151.26ms
step:44/5100 train_loss:5.8549 train_time:5144ms step_avg:151.28ms
step:45/5100 train_loss:5.8095 train_time:5297ms step_avg:151.34ms
step:46/5100 train_loss:5.9779 train_time:5449ms step_avg:151.37ms
step:47/5100 train_loss:5.7638 train_time:5603ms step_avg:151.42ms
step:48/5100 train_loss:5.6291 train_time:5757ms step_avg:151.50ms
step:49/5100 train_loss:5.8346 train_time:5911ms step_avg:151.55ms
step:50/5100 train_loss:5.7183 train_time:6063ms step_avg:151.58ms
step:51/5100 train_loss:5.8558 train_time:6216ms step_avg:151.62ms
step:52/5100 train_loss:5.7231 train_time:6369ms step_avg:151.65ms
step:53/5100 train_loss:5.5794 train_time:6521ms step_avg:151.65ms
step:54/5100 train_loss:5.7221 train_time:6675ms step_avg:151.71ms
step:55/5100 train_loss:5.6005 train_time:6828ms step_avg:151.73ms
step:56/5100 train_loss:5.9398 train_time:6982ms step_avg:151.77ms
step:57/5100 train_loss:5.5927 train_time:7136ms step_avg:151.83ms
step:58/5100 train_loss:5.4627 train_time:7289ms step_avg:151.86ms
step:59/5100 train_loss:5.6116 train_time:7442ms step_avg:151.88ms
step:60/5100 train_loss:5.5761 train_time:7596ms step_avg:151.91ms
step:61/5100 train_loss:5.6906 train_time:7749ms step_avg:151.93ms
step:62/5100 train_loss:5.4456 train_time:7901ms step_avg:151.95ms
step:63/5100 train_loss:5.5463 train_time:8055ms step_avg:151.98ms
step:64/5100 train_loss:5.5327 train_time:8208ms step_avg:152.01ms
step:65/5100 train_loss:5.2273 train_time:8361ms step_avg:152.02ms
step:66/5100 train_loss:5.3407 train_time:8514ms step_avg:152.04ms
step:67/5100 train_loss:5.4952 train_time:8667ms step_avg:152.05ms
step:68/5100 train_loss:5.3736 train_time:8819ms step_avg:152.05ms
step:69/5100 train_loss:5.6261 train_time:8973ms step_avg:152.09ms
step:70/5100 train_loss:5.2765 train_time:9126ms step_avg:152.11ms
step:71/5100 train_loss:5.2919 train_time:9280ms step_avg:152.14ms
step:72/5100 train_loss:5.5050 train_time:9434ms step_avg:152.16ms
step:73/5100 train_loss:5.4406 train_time:9587ms step_avg:152.17ms
step:74/5100 train_loss:5.3141 train_time:9740ms step_avg:152.18ms
step:75/5100 train_loss:5.4444 train_time:9892ms step_avg:152.19ms
step:76/5100 train_loss:5.4133 train_time:10046ms step_avg:152.21ms
step:77/5100 train_loss:5.3648 train_time:10199ms step_avg:152.22ms
step:78/5100 train_loss:5.4578 train_time:10352ms step_avg:152.24ms
step:79/5100 train_loss:5.5200 train_time:10506ms step_avg:152.26ms
step:80/5100 train_loss:5.3211 train_time:10660ms step_avg:152.28ms
step:81/5100 train_loss:5.4260 train_time:10811ms step_avg:152.27ms
step:82/5100 train_loss:5.1893 train_time:10965ms step_avg:152.29ms
step:83/5100 train_loss:5.3617 train_time:11118ms step_avg:152.30ms
step:84/5100 train_loss:5.3141 train_time:11272ms step_avg:152.32ms
step:85/5100 train_loss:5.2966 train_time:11424ms step_avg:152.32ms
step:86/5100 train_loss:5.1611 train_time:11580ms step_avg:152.37ms
step:87/5100 train_loss:5.3763 train_time:11733ms step_avg:152.38ms
step:88/5100 train_loss:5.2845 train_time:11885ms step_avg:152.38ms
step:89/5100 train_loss:5.3295 train_time:12039ms step_avg:152.40ms
step:90/5100 train_loss:5.2873 train_time:12193ms step_avg:152.41ms
step:91/5100 train_loss:5.2253 train_time:12345ms step_avg:152.41ms
step:92/5100 train_loss:5.2012 train_time:12499ms step_avg:152.42ms
step:93/5100 train_loss:5.3525 train_time:12652ms step_avg:152.43ms
step:94/5100 train_loss:5.1548 train_time:12806ms step_avg:152.45ms
step:95/5100 train_loss:5.1634 train_time:12959ms step_avg:152.46ms
step:96/5100 train_loss:5.2021 train_time:13113ms step_avg:152.47ms
step:97/5100 train_loss:5.1079 train_time:13266ms step_avg:152.48ms
step:98/5100 train_loss:5.1923 train_time:13419ms step_avg:152.49ms
step:99/5100 train_loss:5.1158 train_time:13573ms step_avg:152.50ms
step:100/5100 train_loss:5.2396 train_time:13726ms step_avg:152.52ms
step:101/5100 train_loss:5.2138 train_time:13880ms step_avg:152.53ms
step:102/5100 train_loss:5.1164 train_time:14034ms step_avg:152.54ms
step:103/5100 train_loss:5.2165 train_time:14187ms step_avg:152.55ms
step:104/5100 train_loss:5.1526 train_time:14340ms step_avg:152.56ms
step:105/5100 train_loss:5.0218 train_time:14494ms step_avg:152.57ms
step:106/5100 train_loss:5.1203 train_time:14648ms step_avg:152.58ms
step:107/5100 train_loss:5.3267 train_time:14800ms step_avg:152.58ms
step:108/5100 train_loss:5.0854 train_time:14954ms step_avg:152.60ms
step:109/5100 train_loss:4.8868 train_time:15108ms step_avg:152.60ms
step:110/5100 train_loss:5.0813 train_time:15262ms step_avg:152.62ms
step:111/5100 train_loss:5.0444 train_time:15415ms step_avg:152.63ms
step:112/5100 train_loss:5.0087 train_time:15568ms step_avg:152.63ms
step:113/5100 train_loss:5.1094 train_time:15721ms step_avg:152.63ms
step:114/5100 train_loss:5.0430 train_time:15875ms step_avg:152.64ms
step:115/5100 train_loss:4.9000 train_time:16028ms step_avg:152.65ms
step:116/5100 train_loss:5.0611 train_time:16183ms step_avg:152.67ms
step:117/5100 train_loss:4.9731 train_time:16337ms step_avg:152.68ms
step:118/5100 train_loss:4.9182 train_time:16490ms step_avg:152.68ms
step:119/5100 train_loss:5.0629 train_time:16642ms step_avg:152.68ms
step:120/5100 train_loss:5.0233 train_time:16796ms step_avg:152.69ms
step:121/5100 train_loss:4.9470 train_time:16951ms step_avg:152.72ms
step:122/5100 train_loss:4.8487 train_time:17106ms step_avg:152.73ms
step:123/5100 train_loss:4.9770 train_time:17260ms step_avg:152.74ms
step:124/5100 train_loss:4.8247 train_time:17414ms step_avg:152.75ms
step:125/5100 train_loss:5.1450 train_time:17568ms step_avg:152.76ms
step:125/5100 val_loss:4.9669 train_time:17631ms step_avg:153.32ms
step:126/5100 train_loss:5.0153 train_time:17724ms step_avg:152.79ms
step:127/5100 train_loss:4.9507 train_time:17883ms step_avg:152.85ms
step:128/5100 train_loss:5.0129 train_time:18036ms step_avg:152.85ms
step:129/5100 train_loss:4.8857 train_time:18188ms step_avg:152.84ms
step:130/5100 train_loss:5.1978 train_time:18341ms step_avg:152.84ms
step:131/5100 train_loss:4.9433 train_time:18493ms step_avg:152.83ms
step:132/5100 train_loss:4.9528 train_time:18645ms step_avg:152.83ms
step:133/5100 train_loss:4.9133 train_time:18801ms step_avg:152.85ms
step:134/5100 train_loss:4.9434 train_time:18955ms step_avg:152.86ms
step:135/5100 train_loss:4.8356 train_time:19108ms step_avg:152.87ms
step:136/5100 train_loss:4.9590 train_time:19262ms step_avg:152.87ms
step:137/5100 train_loss:4.7379 train_time:19415ms step_avg:152.87ms
step:138/5100 train_loss:4.9000 train_time:19566ms step_avg:152.86ms
step:139/5100 train_loss:4.8433 train_time:19720ms step_avg:152.87ms
step:140/5100 train_loss:4.8878 train_time:19874ms step_avg:152.88ms
step:141/5100 train_loss:4.9478 train_time:20027ms step_avg:152.88ms
step:142/5100 train_loss:4.8241 train_time:20181ms step_avg:152.89ms
step:143/5100 train_loss:4.8760 train_time:20335ms step_avg:152.89ms
step:144/5100 train_loss:4.7360 train_time:20489ms step_avg:152.91ms
step:145/5100 train_loss:4.8745 train_time:20642ms step_avg:152.90ms
step:146/5100 train_loss:4.8265 train_time:20796ms step_avg:152.91ms
step:147/5100 train_loss:4.7005 train_time:20948ms step_avg:152.91ms
step:148/5100 train_loss:4.8448 train_time:21102ms step_avg:152.91ms
step:149/5100 train_loss:4.8398 train_time:21255ms step_avg:152.92ms
step:150/5100 train_loss:4.8679 train_time:21409ms step_avg:152.92ms
step:151/5100 train_loss:4.9167 train_time:21563ms step_avg:152.93ms
step:152/5100 train_loss:4.7985 train_time:21716ms step_avg:152.93ms
step:153/5100 train_loss:4.7928 train_time:21869ms step_avg:152.93ms
step:154/5100 train_loss:4.8811 train_time:22022ms step_avg:152.93ms
step:155/5100 train_loss:4.8371 train_time:22177ms step_avg:152.94ms
step:156/5100 train_loss:4.7923 train_time:22329ms step_avg:152.94ms
step:157/5100 train_loss:4.8188 train_time:22483ms step_avg:152.94ms
step:158/5100 train_loss:4.9365 train_time:22636ms step_avg:152.95ms
step:159/5100 train_loss:4.7256 train_time:22789ms step_avg:152.95ms
step:160/5100 train_loss:4.8005 train_time:22944ms step_avg:152.96ms
step:161/5100 train_loss:4.6270 train_time:23096ms step_avg:152.96ms
step:162/5100 train_loss:4.8134 train_time:23249ms step_avg:152.96ms
step:163/5100 train_loss:4.8456 train_time:23402ms step_avg:152.95ms
step:164/5100 train_loss:4.8269 train_time:23555ms step_avg:152.96ms
step:165/5100 train_loss:4.6422 train_time:23709ms step_avg:152.96ms
step:166/5100 train_loss:4.7655 train_time:23863ms step_avg:152.97ms
step:167/5100 train_loss:4.9010 train_time:24016ms step_avg:152.97ms
step:168/5100 train_loss:4.6904 train_time:24170ms step_avg:152.98ms
step:169/5100 train_loss:4.7889 train_time:24322ms step_avg:152.97ms
step:170/5100 train_loss:4.6299 train_time:24477ms step_avg:152.98ms
step:171/5100 train_loss:4.5340 train_time:24630ms step_avg:152.98ms
step:172/5100 train_loss:4.6949 train_time:24783ms step_avg:152.98ms
step:173/5100 train_loss:4.6711 train_time:24937ms step_avg:152.99ms
step:174/5100 train_loss:4.7388 train_time:25090ms step_avg:152.99ms
step:175/5100 train_loss:4.8856 train_time:25243ms step_avg:152.99ms
step:176/5100 train_loss:4.7307 train_time:25395ms step_avg:152.98ms
step:177/5100 train_loss:4.5852 train_time:25549ms step_avg:152.99ms
step:178/5100 train_loss:4.5581 train_time:25701ms step_avg:152.98ms
step:179/5100 train_loss:4.6316 train_time:25855ms step_avg:152.99ms
step:180/5100 train_loss:4.6384 train_time:26009ms step_avg:152.99ms
step:181/5100 train_loss:4.6272 train_time:26163ms step_avg:153.00ms
step:182/5100 train_loss:4.7598 train_time:26316ms step_avg:153.00ms
step:183/5100 train_loss:4.6281 train_time:26469ms step_avg:153.00ms
step:184/5100 train_loss:4.5748 train_time:26621ms step_avg:153.00ms
step:185/5100 train_loss:4.5907 train_time:26775ms step_avg:153.00ms
step:186/5100 train_loss:4.7187 train_time:26928ms step_avg:153.00ms
step:187/5100 train_loss:4.6292 train_time:27082ms step_avg:153.01ms
step:188/5100 train_loss:4.8232 train_time:27235ms step_avg:153.01ms
step:189/5100 train_loss:4.6354 train_time:27487ms step_avg:153.56ms
step:190/5100 train_loss:4.5647 train_time:27775ms step_avg:154.31ms
step:191/5100 train_loss:4.7053 train_time:27925ms step_avg:154.28ms
step:192/5100 train_loss:4.5519 train_time:28077ms step_avg:154.27ms
step:193/5100 train_loss:4.4761 train_time:28230ms step_avg:154.26ms
step:194/5100 train_loss:4.6939 train_time:28383ms step_avg:154.25ms
step:195/5100 train_loss:4.6267 train_time:28535ms step_avg:154.24ms
step:196/5100 train_loss:4.8096 train_time:28688ms step_avg:154.23ms
step:197/5100 train_loss:4.6832 train_time:28841ms step_avg:154.23ms
step:198/5100 train_loss:4.5200 train_time:28996ms step_avg:154.24ms
step:199/5100 train_loss:4.5972 train_time:29150ms step_avg:154.23ms
step:200/5100 train_loss:4.4624 train_time:29302ms step_avg:154.22ms
step:201/5100 train_loss:4.5590 train_time:29455ms step_avg:154.22ms
step:202/5100 train_loss:4.4535 train_time:29608ms step_avg:154.21ms
step:203/5100 train_loss:4.7074 train_time:29761ms step_avg:154.20ms
step:204/5100 train_loss:4.5699 train_time:29915ms step_avg:154.20ms
step:205/5100 train_loss:4.5984 train_time:30069ms step_avg:154.20ms
step:206/5100 train_loss:4.7220 train_time:30222ms step_avg:154.19ms
step:207/5100 train_loss:4.3753 train_time:30376ms step_avg:154.19ms
step:208/5100 train_loss:4.5346 train_time:30529ms step_avg:154.19ms
step:209/5100 train_loss:4.5112 train_time:30682ms step_avg:154.18ms
step:210/5100 train_loss:4.6699 train_time:30835ms step_avg:154.18ms
step:211/5100 train_loss:4.5814 train_time:30988ms step_avg:154.17ms
step:212/5100 train_loss:4.4721 train_time:31140ms step_avg:154.16ms
step:213/5100 train_loss:4.5835 train_time:31294ms step_avg:154.16ms
step:214/5100 train_loss:4.4429 train_time:31448ms step_avg:154.16ms
step:215/5100 train_loss:4.5102 train_time:31600ms step_avg:154.15ms
step:216/5100 train_loss:4.3772 train_time:31754ms step_avg:154.14ms
step:217/5100 train_loss:4.4848 train_time:31906ms step_avg:154.14ms
step:218/5100 train_loss:4.4521 train_time:32060ms step_avg:154.14ms
step:219/5100 train_loss:4.4727 train_time:32214ms step_avg:154.13ms
step:220/5100 train_loss:4.4671 train_time:32367ms step_avg:154.13ms
step:221/5100 train_loss:4.5000 train_time:32520ms step_avg:154.12ms
step:222/5100 train_loss:4.5143 train_time:32673ms step_avg:154.12ms
step:223/5100 train_loss:4.4404 train_time:32826ms step_avg:154.11ms
step:224/5100 train_loss:4.4540 train_time:32980ms step_avg:154.11ms
step:225/5100 train_loss:4.6417 train_time:33134ms step_avg:154.11ms
step:226/5100 train_loss:4.3033 train_time:33287ms step_avg:154.11ms
step:227/5100 train_loss:4.3602 train_time:33442ms step_avg:154.11ms
step:228/5100 train_loss:4.3672 train_time:33595ms step_avg:154.11ms
step:229/5100 train_loss:4.5179 train_time:33748ms step_avg:154.10ms
step:230/5100 train_loss:4.3141 train_time:33901ms step_avg:154.10ms
step:231/5100 train_loss:4.4583 train_time:34055ms step_avg:154.10ms
step:232/5100 train_loss:4.3263 train_time:34208ms step_avg:154.09ms
step:233/5100 train_loss:4.3273 train_time:34362ms step_avg:154.09ms
step:234/5100 train_loss:4.4929 train_time:34515ms step_avg:154.09ms
step:235/5100 train_loss:4.3778 train_time:34668ms step_avg:154.08ms
step:236/5100 train_loss:4.2735 train_time:34820ms step_avg:154.07ms
step:237/5100 train_loss:4.4817 train_time:34974ms step_avg:154.07ms
step:238/5100 train_loss:4.4401 train_time:35127ms step_avg:154.06ms
step:239/5100 train_loss:4.3106 train_time:35280ms step_avg:154.06ms
step:240/5100 train_loss:4.4647 train_time:35434ms step_avg:154.06ms
step:241/5100 train_loss:4.4568 train_time:35588ms step_avg:154.06ms
step:242/5100 train_loss:4.3369 train_time:35742ms step_avg:154.06ms
step:243/5100 train_loss:4.5240 train_time:35896ms step_avg:154.06ms
step:244/5100 train_loss:4.3587 train_time:36048ms step_avg:154.05ms
step:245/5100 train_loss:4.3904 train_time:36201ms step_avg:154.05ms
step:246/5100 train_loss:4.4742 train_time:36354ms step_avg:154.04ms
step:247/5100 train_loss:4.4093 train_time:36508ms step_avg:154.04ms
step:248/5100 train_loss:4.3459 train_time:36660ms step_avg:154.04ms
step:249/5100 train_loss:4.4756 train_time:36814ms step_avg:154.03ms
step:250/5100 train_loss:4.2456 train_time:36967ms step_avg:154.03ms
step:250/5100 val_loss:4.3420 train_time:37031ms step_avg:154.30ms
step:251/5100 train_loss:4.2975 train_time:37122ms step_avg:154.03ms
step:252/5100 train_loss:4.4059 train_time:37282ms step_avg:154.06ms
step:253/5100 train_loss:4.4499 train_time:37436ms step_avg:154.06ms
step:254/5100 train_loss:4.2720 train_time:37587ms step_avg:154.04ms
step:255/5100 train_loss:4.2149 train_time:37740ms step_avg:154.04ms
step:256/5100 train_loss:4.3911 train_time:37892ms step_avg:154.03ms
step:257/5100 train_loss:4.3103 train_time:38045ms step_avg:154.03ms
step:258/5100 train_loss:4.3216 train_time:38199ms step_avg:154.03ms
step:259/5100 train_loss:4.2828 train_time:38353ms step_avg:154.03ms
step:260/5100 train_loss:4.3226 train_time:38506ms step_avg:154.03ms
step:261/5100 train_loss:4.3673 train_time:38659ms step_avg:154.02ms
step:262/5100 train_loss:4.3305 train_time:38811ms step_avg:154.01ms
step:263/5100 train_loss:4.2978 train_time:38964ms step_avg:154.01ms
step:264/5100 train_loss:4.2126 train_time:39117ms step_avg:154.00ms
step:265/5100 train_loss:4.2962 train_time:39271ms step_avg:154.00ms
step:266/5100 train_loss:4.1531 train_time:39424ms step_avg:154.00ms
step:267/5100 train_loss:4.2122 train_time:39577ms step_avg:154.00ms
step:268/5100 train_loss:4.2295 train_time:39731ms step_avg:153.99ms
step:269/5100 train_loss:4.2357 train_time:39883ms step_avg:153.99ms
step:270/5100 train_loss:4.1582 train_time:40036ms step_avg:153.98ms
step:271/5100 train_loss:4.3940 train_time:40188ms step_avg:153.98ms
step:272/5100 train_loss:4.2917 train_time:40341ms step_avg:153.97ms
step:273/5100 train_loss:4.1918 train_time:40495ms step_avg:153.97ms
step:274/5100 train_loss:4.2445 train_time:40649ms step_avg:153.97ms
step:275/5100 train_loss:4.3226 train_time:40803ms step_avg:153.97ms
step:276/5100 train_loss:4.3393 train_time:40956ms step_avg:153.97ms
step:277/5100 train_loss:4.5242 train_time:41109ms step_avg:153.97ms
step:278/5100 train_loss:4.3126 train_time:41261ms step_avg:153.96ms
step:279/5100 train_loss:4.3895 train_time:41414ms step_avg:153.96ms
step:280/5100 train_loss:4.2820 train_time:41567ms step_avg:153.95ms
step:281/5100 train_loss:4.3993 train_time:41719ms step_avg:153.95ms
step:282/5100 train_loss:4.2367 train_time:41874ms step_avg:153.95ms
step:283/5100 train_loss:4.2551 train_time:42027ms step_avg:153.95ms
step:284/5100 train_loss:4.1814 train_time:42180ms step_avg:153.94ms
step:285/5100 train_loss:4.3337 train_time:42334ms step_avg:153.94ms
step:286/5100 train_loss:4.3377 train_time:42486ms step_avg:153.94ms
step:287/5100 train_loss:4.3617 train_time:42638ms step_avg:153.93ms
step:288/5100 train_loss:4.1948 train_time:42791ms step_avg:153.93ms
step:289/5100 train_loss:4.2898 train_time:42944ms step_avg:153.92ms
step:290/5100 train_loss:4.1467 train_time:43097ms step_avg:153.92ms
step:291/5100 train_loss:4.1432 train_time:43251ms step_avg:153.92ms
step:292/5100 train_loss:4.2244 train_time:43404ms step_avg:153.92ms
step:293/5100 train_loss:4.1343 train_time:43557ms step_avg:153.91ms
step:294/5100 train_loss:4.1883 train_time:43711ms step_avg:153.91ms
step:295/5100 train_loss:4.2213 train_time:43863ms step_avg:153.91ms
step:296/5100 train_loss:4.1006 train_time:44015ms step_avg:153.90ms
step:297/5100 train_loss:4.1221 train_time:44169ms step_avg:153.90ms
step:298/5100 train_loss:4.1272 train_time:44321ms step_avg:153.89ms
step:299/5100 train_loss:4.2334 train_time:44475ms step_avg:153.89ms
step:300/5100 train_loss:4.0936 train_time:44629ms step_avg:153.89ms
step:301/5100 train_loss:4.2395 train_time:44782ms step_avg:153.89ms
step:302/5100 train_loss:4.2444 train_time:44936ms step_avg:153.89ms
step:303/5100 train_loss:4.1797 train_time:45088ms step_avg:153.88ms
step:304/5100 train_loss:4.2444 train_time:45241ms step_avg:153.88ms
step:305/5100 train_loss:4.2217 train_time:45394ms step_avg:153.88ms
step:306/5100 train_loss:4.6954 train_time:45547ms step_avg:153.88ms
step:307/5100 train_loss:4.1937 train_time:45699ms step_avg:153.87ms
step:308/5100 train_loss:4.0995 train_time:45853ms step_avg:153.87ms
step:309/5100 train_loss:4.2599 train_time:46005ms step_avg:153.86ms
step:310/5100 train_loss:4.1046 train_time:46158ms step_avg:153.86ms
step:311/5100 train_loss:4.3338 train_time:46311ms step_avg:153.86ms
step:312/5100 train_loss:4.1882 train_time:46465ms step_avg:153.86ms
step:313/5100 train_loss:4.1218 train_time:46616ms step_avg:153.85ms
step:314/5100 train_loss:4.2396 train_time:46771ms step_avg:153.85ms
step:315/5100 train_loss:4.3367 train_time:46923ms step_avg:153.85ms
step:316/5100 train_loss:4.2161 train_time:47077ms step_avg:153.85ms
step:317/5100 train_loss:4.0477 train_time:47231ms step_avg:153.85ms
step:318/5100 train_loss:4.1242 train_time:47384ms step_avg:153.84ms
step:319/5100 train_loss:4.1575 train_time:47536ms step_avg:153.84ms
step:320/5100 train_loss:4.1365 train_time:47689ms step_avg:153.84ms
step:321/5100 train_loss:4.2392 train_time:47842ms step_avg:153.83ms
step:322/5100 train_loss:4.1963 train_time:47995ms step_avg:153.83ms
step:323/5100 train_loss:4.1661 train_time:48149ms step_avg:153.83ms
step:324/5100 train_loss:4.2576 train_time:48303ms step_avg:153.83ms
step:325/5100 train_loss:4.2126 train_time:48456ms step_avg:153.83ms
step:326/5100 train_loss:4.2788 train_time:48610ms step_avg:153.83ms
step:327/5100 train_loss:4.1298 train_time:48762ms step_avg:153.82ms
step:328/5100 train_loss:4.6276 train_time:48915ms step_avg:153.82ms
step:329/5100 train_loss:4.3128 train_time:49068ms step_avg:153.82ms
step:330/5100 train_loss:4.0488 train_time:49220ms step_avg:153.81ms
step:331/5100 train_loss:4.0034 train_time:49375ms step_avg:153.82ms
step:332/5100 train_loss:4.2231 train_time:49528ms step_avg:153.81ms
step:333/5100 train_loss:4.1383 train_time:49681ms step_avg:153.81ms
step:334/5100 train_loss:4.1230 train_time:49835ms step_avg:153.81ms
step:335/5100 train_loss:4.0844 train_time:49986ms step_avg:153.80ms
step:336/5100 train_loss:4.2570 train_time:50138ms step_avg:153.80ms
step:337/5100 train_loss:4.1924 train_time:50291ms step_avg:153.79ms
step:338/5100 train_loss:4.6707 train_time:50444ms step_avg:153.79ms
step:339/5100 train_loss:4.1788 train_time:50597ms step_avg:153.79ms
step:340/5100 train_loss:4.1239 train_time:50751ms step_avg:153.79ms
step:341/5100 train_loss:4.1599 train_time:50904ms step_avg:153.79ms
step:342/5100 train_loss:4.0778 train_time:51057ms step_avg:153.79ms
step:343/5100 train_loss:4.0494 train_time:51209ms step_avg:153.78ms
step:344/5100 train_loss:4.0997 train_time:51361ms step_avg:153.78ms
step:345/5100 train_loss:4.2330 train_time:51514ms step_avg:153.77ms
step:346/5100 train_loss:4.0750 train_time:51666ms step_avg:153.77ms
step:347/5100 train_loss:4.0057 train_time:51818ms step_avg:153.76ms
step:348/5100 train_loss:4.0549 train_time:51973ms step_avg:153.77ms
step:349/5100 train_loss:4.0957 train_time:52125ms step_avg:153.76ms
step:350/5100 train_loss:4.0547 train_time:52278ms step_avg:153.76ms
step:351/5100 train_loss:3.7710 train_time:52431ms step_avg:153.76ms
step:352/5100 train_loss:4.0464 train_time:52584ms step_avg:153.75ms
step:353/5100 train_loss:4.3910 train_time:52736ms step_avg:153.75ms
step:354/5100 train_loss:3.8918 train_time:52889ms step_avg:153.75ms
step:355/5100 train_loss:4.1504 train_time:53042ms step_avg:153.75ms
step:356/5100 train_loss:4.0241 train_time:53195ms step_avg:153.74ms
step:357/5100 train_loss:4.1256 train_time:53348ms step_avg:153.74ms
step:358/5100 train_loss:4.0636 train_time:53502ms step_avg:153.74ms
step:359/5100 train_loss:4.0779 train_time:53655ms step_avg:153.74ms
step:360/5100 train_loss:4.1309 train_time:53808ms step_avg:153.74ms
step:361/5100 train_loss:3.6846 train_time:53960ms step_avg:153.73ms
step:362/5100 train_loss:4.2538 train_time:54114ms step_avg:153.73ms
step:363/5100 train_loss:4.1440 train_time:54266ms step_avg:153.73ms
step:364/5100 train_loss:4.0703 train_time:54418ms step_avg:153.72ms
step:365/5100 train_loss:3.9769 train_time:54572ms step_avg:153.72ms
step:366/5100 train_loss:4.1383 train_time:54725ms step_avg:153.72ms
step:367/5100 train_loss:4.1000 train_time:54879ms step_avg:153.72ms
step:368/5100 train_loss:4.0840 train_time:55033ms step_avg:153.72ms
step:369/5100 train_loss:4.0698 train_time:55185ms step_avg:153.72ms
step:370/5100 train_loss:3.9639 train_time:55337ms step_avg:153.71ms
step:371/5100 train_loss:4.1196 train_time:55490ms step_avg:153.71ms
step:372/5100 train_loss:3.9940 train_time:55643ms step_avg:153.71ms
step:373/5100 train_loss:3.9145 train_time:55795ms step_avg:153.70ms
step:374/5100 train_loss:4.1327 train_time:55948ms step_avg:153.70ms
step:375/5100 train_loss:4.0621 train_time:56101ms step_avg:153.70ms
step:375/5100 val_loss:4.0605 train_time:56167ms step_avg:153.88ms
step:376/5100 train_loss:4.0358 train_time:56261ms step_avg:153.72ms
step:377/5100 train_loss:4.0955 train_time:56417ms step_avg:153.72ms
step:378/5100 train_loss:4.0143 train_time:56664ms step_avg:153.98ms
step:379/5100 train_loss:4.0625 train_time:56824ms step_avg:154.00ms
step:380/5100 train_loss:4.1093 train_time:57099ms step_avg:154.32ms
step:381/5100 train_loss:4.1681 train_time:57249ms step_avg:154.31ms
step:382/5100 train_loss:4.0795 train_time:57401ms step_avg:154.30ms
step:383/5100 train_loss:4.0486 train_time:57553ms step_avg:154.30ms
step:384/5100 train_loss:4.0117 train_time:57704ms step_avg:154.29ms
step:385/5100 train_loss:4.0987 train_time:57857ms step_avg:154.28ms
step:386/5100 train_loss:4.0053 train_time:58009ms step_avg:154.28ms
step:387/5100 train_loss:4.1190 train_time:58163ms step_avg:154.28ms
step:388/5100 train_loss:4.3082 train_time:58317ms step_avg:154.28ms
step:389/5100 train_loss:4.0209 train_time:58469ms step_avg:154.27ms
step:390/5100 train_loss:4.0149 train_time:58621ms step_avg:154.26ms
step:391/5100 train_loss:4.1150 train_time:58773ms step_avg:154.26ms
step:392/5100 train_loss:4.0298 train_time:58924ms step_avg:154.25ms
step:393/5100 train_loss:4.1390 train_time:59077ms step_avg:154.25ms
step:394/5100 train_loss:3.9758 train_time:59230ms step_avg:154.25ms
step:395/5100 train_loss:4.1108 train_time:59384ms step_avg:154.24ms
step:396/5100 train_loss:3.8561 train_time:59537ms step_avg:154.24ms
step:397/5100 train_loss:4.0557 train_time:59689ms step_avg:154.24ms
step:398/5100 train_loss:4.1114 train_time:59842ms step_avg:154.23ms