එම්එල්පී මික්සර් අත්හදා බැලීම

12from labml import experiment
13from labml.configs import option
14from labml_nn.transformers import TransformerConfigs
15from labml_nn.transformers.configs import FeedForwardConfigs
16from labml_nn.transformers.mlm.experiment import TransformerMLM, Configs as MLMConfigs

19class Configs(MLMConfigs):

29    mix_mlp: FeedForwardConfigs

32@option(Configs.mix_mlp)
33def _mix_mlp_configs(c: Configs):

38    conf = FeedForwardConfigs()

40    conf.d_model = c.seq_len

42    conf.activation = 'GELU'

45    return conf

48@option(Configs.transformer)
49def _transformer_configs(c: Configs):

56    conf = TransformerConfigs()

58    conf.n_src_vocab = c.n_tokens
59    conf.n_tgt_vocab = c.n_tokens

61    conf.d_model = c.d_model

63    from labml_nn.transformers.mlp_mixer import MLPMixer
64    conf.encoder_attn = MLPMixer(c.mix_mlp.ffn)

67    return conf

70def main():

72    experiment.create(name="mlp_mixer_mlm")

74    conf = Configs()

76    experiment.configs(conf, {

78        'batch_size': 64,

81        'seq_len': 32,

84        'epochs': 1024,

87        'inner_iterations': 1,

90        'd_model': 128,
91        'transformer.ffn.d_ff': 256,
92        'transformer.n_heads': 8,
93        'transformer.n_layers': 6,
94        'transformer.ffn.activation': 'GELU',

97        'mix_mlp.d_ff': 128,

100        'optimizer.optimizer': 'Noam',
101        'optimizer.learning_rate': 1.,
102    })

105    experiment.add_pytorch_models({'model': conf.model})

108    with experiment.start():

110        conf.run()

114if __name__ == '__main__':
115    main()