");--av-icon-instagram:url("data:image/svg+xml;charset=utf-8, ");--av-icon-linkedin:url("data:image/svg+xml;charset=utf-8, ");--av-icon-pinterest:url("data:image/svg+xml;charset=utf-8, ");--av-icon-print:url("data:image/svg+xml;charset=utf-8, ");--av-icon-rss:url("data:image/svg+xml;charset=utf-8, ");--av-icon-telegram:url("data:image/svg+xml;charset=utf-8, ");--av-icon-twitter:url("data:image/svg+xml;charset=utf-8, ");--av-icon-whatsapp:url("data:image/svg+xml;charset=utf-8, ");--av-icon-youtube:url("data:image/svg+xml;charset=utf-8, ");--av-icon-tiktok:url("data:image/svg+xml;charset=utf-8, ");--av-icon-discord:url("data:image/svg+xml;charset=utf-8, ");--av-facebook-color:#1877f2;--av-instagram-color:#f00075;--av-linkedin-color:#0d66c2;--av-pinterest-color:#e60122;--av-telegram-icon:#2aabee;--av-twitter-color:#000;--av-whatsapp-color:#25d366;--av-youtube-color:red}.av-social-share{align-items:center;display:flex;flex-wrap:wrap;gap:8px;justify-content:center;margin:1em 0;overflow:hidden;padding:0}@media screen and (min-width:480px){.av-social-share{justify-content:flex-start}}#avConsent:not([class*=-amphtml-consent-ui-iframe-fullscreen]){bottom:auto;box-shadow:0 0 20px 0 #000;max-height:60vh;top:0}@keyframes av-affiliation-fade{0%{opacity:.4}to{opacity:1}}body{background:#fff;color:#333;font-family:Krub;font-size:16px;font-weight:400;line-height:1.75;position:relative;margin:0}.amp-wp-header,.amp-wp-article,.amp-wp-footer{max-width:600px;margin:auto}p,ul{margin:0 0 1.5em;padding:0}a,a:visited{color:#ce5c5c;text-decoration:underline}a:hover,a:active,a:focus{color:#ce5c5c;text-decoration:underline}h1,h2,h3,.amp-wp-article-content h1,.amp-wp-article-content h2,.amp-wp-article-content h3{color:#1a1a1a;font-family:Lora}h1,h2,h3{font-family:Lora;font-weight:700;letter-spacing:.5px;line-height:1.25;margin:0 auto 1rem;padding:0;color:#1a1a1a}h1.amp-wp-title{color:#222;font-family:Lora;font-weight:400;display:block;flex:1 0 100%;margin:0 0 1rem;width:100%;font-size:1.5rem;text-align:center}.amp-wp-header div,.amp-wp-title,.amp-wp-footer p{font-family:-apple-system,BlinkMacSystemFont,"Segoe UI","Roboto","Oxygen-Sans","Ubuntu","Cantarell","Helvetica Neue",sans-serif}.amp-wp-header{padding:0;margin:auto;height:50px;line-height:50px;text-align:center;white-space:nowrap;font-size:18px;font-weight:700;background:#ce5c5c;color:#fff}.amp-wp-header{color:#fff;font-size:1em;font-weight:400}.amp-wp-header-container{position:relative}.amp-wp-header a{color:#fff;text-decoration:none}.amp-wp-header .header-title{padding:0 60px;letter-spacing:.5px}.amp-wp-header .header-title a{font-family:Lilita One}.amp-wp-header div a{color:#fff;text-decoration:none;text-align:center;font-size:18px}.amp-wp-header .header-menu-toggle{width:30px;height:30px;padding:5px 10px;position:absolute;display:block;top:5px;left:5px;background:0 0;line-height:30px;z-index:10}.amp-wp-header .header-menu-toggle button.menu-toggle{font-size:21px;line-height:1;font-weight:400;background:transparent;color:#fff;border:none;padding:0;outline:none}.amp-wp-header .header-menu-toggle svg{fill:#fff}#site-menu{width:100%;background-color:#ce5c5c;color:#fff;text-align:left;padding-left:5px;padding-right:5px}#site-menu a,#site-menu a:visited{display:block;margin:5px 0;padding:5px 0;color:#fff;font-weight:500;font-size:14px;text-transform:uppercase;text-decoration:none;line-height:28px}#site-menu ul{padding:5px 0;list-style-type:none;position:relative}#site-menu ul li{padding:0;margin:0 15px;border-bottom:1px solid rgba(255,255,255,.15)}#site-menu ul li.sidebar-close{position:relative;height:50px;border:0}#site-menu .menu-close{position:absolute;right:-5px;top:3px;width:40px;height:40px;background:transparent;border:0;padding:5px;outline:none}#site-menu .menu-close span{position:relative;display:block;margin:auto;background-color:transparent;content:"";height:2px;width:24px;opacity:1}#site-menu .menu-close span:before{background-color:#fff;content:"";height:2px;width:24px;left:0;position:absolute;top:-6px;transform-origin:center center 0;-webkit-transform:translate(0,6px) rotate(-45deg);transform:translate(0,6px) rotate(-45deg)}#site-menu .menu-close span:after{background-color:#fff;bottom:-6px;content:"";height:2px;width:24px;position:absolute;left:0;transform-origin:center center 0;-webkit-transform:translate(0,-6px) rotate(45deg);transform:translate(0,-6px) rotate(45deg)}.amp-wp-article{font-weight:400;margin:1.5em auto;overflow-wrap:break-word;overflow-x:hidden}.amp-wp-article-header{align-items:center;align-content:stretch;display:flex;flex-wrap:wrap;justify-content:space-between;margin:1.5em 10px 0}.amp-wp-title{color:#1a1a1a;display:block;flex:1 0 100%;font-weight:900;margin:0 0 .625em;width:100%}.amp-wp-article-content{margin:0 10px}.amp-wp-article-content ul{margin:1.5em 0 1.5em 1.5em}.amp-wp-article-content amp-img{margin:0 auto}.amp-wp-footer{margin:1.5em auto;border-top:1px solid rgba(0,0,0,.1)}.amp-wp-footer div{position:relative;margin:0 20px;padding:1.5em 0}.amp-wp-footer h2{font-size:1em;line-height:1.375em;text-align:center;margin:0 0 .5em;color:#222}.amp-wp-footer p{color:#888;font-size:.8em;line-height:1.5em;margin:0 85px 0 0}.amp-wp-footer a{text-decoration:none}.amp-wp-footer .av-credit-link{font-size:11px;clear:both;text-align:center;padding:10px 0;width:100%;margin:0}.amp-related-posts{margin:0 5px}.amp-related-posts amp-img{object-fit:contain}.amp-related-posts-cont{text-align:center}.amp-related-posts h2{color:#222;font-size:1.5rem;margin:1.5rem 0;text-align:center}.amp-related-posts h3{color:#222;margin-bottom:1rem;padding:1rem 0 0;font-size:1.125rem}.amp-related-posts-cont ul{list-style:none;display:flex;flex-wrap:wrap;flex-direction:row}.amp-related-posts-cont ul li{margin-bottom:1rem;width:50%;padding:0 5px;box-sizing:border-box}.amp-related-posts a{font-size:1.125rem;text-decoration:none}.av-social-share{text-align:center}.av-social-share amp-social-share{margin:0 10px}@media screen and (min-width:360px){.amp-wp-article-content,.amp-wp-article-header{margin-left:20px;margin-right:20px}.amp-related-posts{margin-left:10px;margin-right:10px}.amp-related-posts-cont ul li{padding-left:10px;padding-right:10px}}:root:not(#_):not(#_):not(#_):not(#_):not(#_) .amp-wp-f8c7f22{position:relative;font-size:12px;clear:both;text-align:center}:root:not(#_):not(#_):not(#_):not(#_):not(#_):not(#_):not(#_):not(#_):not(#_):not(#_):not(#_):not(#_):not(#_):not(#_):not(#_):not(#_):not(#_) .amp-wp-f8c7f22{padding:8px;margin:0}
Tensor and Parallel AI training in Colossus AI - GXVTRONICS Tensor and Parallel AI training in Colossus AI Colossus is a ‘colossal’ parallel distrubuted tools collection for big datacenter, which rely heavily on parallelism. Some of the most important characteristics are:
Data Parallelism Pipeline Parallelism 1D, 2D, 2.5D, 3D tensor parallelism Sequence parallelism Friendly trainer and engine Extensible for new parallelism Mixed Precision Training Zero Redundancy Optimizer (ZeRO) This toolset include GPT-3 and Bert, autoregresive models to produce human-like texts.
Links
https://analyticsindiamag.com/a-guide-to-parallel-deep-learning-with-colossal-ai/